From b075d3c8fdf6dece1c66d901a04a651509b2e7fd Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 30 Apr 2023 15:13:56 +0300 Subject: [PATCH 1/6] Intel ARC Support --- modules/cmd_args.py | 1 + modules/codeformer_model.py | 6 +- modules/devices.py | 33 ++++++++--- modules/memmon.py | 55 ++++++++++++++----- modules/processing.py | 12 +++- modules/sd_hijack_optimizations.py | 55 ++++++++++++++----- modules/sd_hijack_unet.py | 3 +- modules/sd_models.py | 1 - modules/shared.py | 4 +- .../textual_inversion/textual_inversion.py | 6 +- setup.py | 18 +++++- 11 files changed, 153 insertions(+), 41 deletions(-) diff --git a/modules/cmd_args.py b/modules/cmd_args.py index 9b8438800..fdc9ab240 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -23,6 +23,7 @@ parser.add_argument("--allow-code", action='store_true', help="Allow custom scri parser.add_argument("--share", action='store_true', help="Enable to make the UI accessible through Gradio site") parser.add_argument("--enable-insecure", action='store_true', help="Enable extensions tab regardless of other options") parser.add_argument("--use-cpu", nargs='+', help="Force use CPU for specified modules", default=[], type=str.lower) +parser.add_argument("--use-ipex", action='store_true', help="Force use Intel OneAPI XPU backend") parser.add_argument("--listen", action='store_true', help="Launch web server using public IP address") parser.add_argument("--port", type=int, help="Launch web server with given server port", default=None) parser.add_argument("--hide-ui-dir-config", action='store_true', help="Hide directory configuration from UI", default=False) diff --git a/modules/codeformer_model.py b/modules/codeformer_model.py index cbe06ec1e..5217f69db 100644 --- a/modules/codeformer_model.py +++ b/modules/codeformer_model.py @@ -103,7 +103,11 @@ def setup_model(dirname): output = self.net(cropped_face_t, w=w if w is not None else shared.opts.code_former_weight, adain=True)[0] restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1)) del output - torch.cuda.empty_cache() + from modules import shared + if shared.cmd_opts.use_ipex: + torch.xpu.empty_cache() + else: + torch.cuda.empty_cache() except Exception as error: print(f'\tFailed inference for CodeFormer: {error}', file=sys.stderr) restored_face = tensor2img(cropped_face_t, rgb2bgr=True, min_max=(-1, 1)) diff --git a/modules/devices.py b/modules/devices.py index e317d91f4..3606597d3 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -22,9 +22,13 @@ def extract_device_id(args, name): def get_cuda_device_string(): from modules import shared - if shared.cmd_opts.device_id is not None: - return f"cuda:{shared.cmd_opts.device_id}" - return "cuda" + if shared.cmd_opts.use_ipex: + return "xpu" + else: + from modules import shared + if shared.cmd_opts.device_id is not None: + return f"cuda:{shared.cmd_opts.device_id}" + return "cuda" def get_dml_device_string(): @@ -35,7 +39,10 @@ def get_dml_device_string(): def get_optimal_device_name(): - if torch.cuda.is_available(): + from modules import shared + if shared.cmd_opts.use_ipex: + return "xpu" + elif torch.cuda.is_available(): return get_cuda_device_string() if has_mps(): return "mps" @@ -61,7 +68,11 @@ def get_device_for(task): def torch_gc(): - if torch.cuda.is_available(): + from modules import shared + if shared.cmd_opts.use_ipex: + with torch.xpu.device("xpu"): + torch.xpu.empty_cache() + elif torch.cuda.is_available(): with torch.cuda.device(get_cuda_device_string()): torch.cuda.empty_cache() torch.cuda.ipc_collect() @@ -137,11 +148,19 @@ def autocast(disable=False): return contextlib.nullcontext() if dtype == torch.float32 or shared.cmd_opts.precision == "Full": return contextlib.nullcontext() - return torch.autocast("cuda") + from modules import shared + if shared.cmd_opts.use_ipex: + return torch.xpu.amp.autocast(enabled=True, dtype=dtype, cache_enabled=False) + else: + return torch.autocast("cuda") def without_autocast(disable=False): - return torch.autocast("cuda", enabled=False) if torch.is_autocast_enabled() and not disable else contextlib.nullcontext() + from modules import shared + if shared.cmd_opts.use_ipex: + return torch.autocast("xpu", enabled=False) if torch.is_autocast_enabled() and not disable else contextlib.nullcontext() + else: + return torch.autocast("cuda", enabled=False) if torch.is_autocast_enabled() and not disable else contextlib.nullcontext() class NansException(Exception): diff --git a/modules/memmon.py b/modules/memmon.py index 9b013e6b4..4ceb29a37 100644 --- a/modules/memmon.py +++ b/modules/memmon.py @@ -19,26 +19,44 @@ class MemUsageMonitor(threading.Thread): self.daemon = True self.run_flag = threading.Event() self.data = defaultdict(int) - if not torch.cuda.is_available(): + from modules import shared + if not torch.cuda.is_available() or not shared.cmd_opts.use_ipex: self.disabled = True else: - try: - self.cuda_mem_get_info() - torch.cuda.memory_stats(self.device) - except Exception as e: # AMD or whatever - print(f"Torch exception: {e}") - self.disabled = True + if shared.cmd_opts.use_ipex: + try: + self.cuda_mem_get_info() + torch.cuda.memory_stats("xpu") + except Exception as e: # AMD or whatever + print(f"Torch exception: {e}") + self.disabled = True + + else: + try: + self.cuda_mem_get_info() + torch.cuda.memory_stats(self.device) + except Exception as e: # AMD or whatever + print(f"Torch exception: {e}") + self.disabled = True def cuda_mem_get_info(self): - index = self.device.index if self.device.index is not None else torch.cuda.current_device() - return torch.cuda.mem_get_info(index) + from modules import shared + if shared.cmd_opts.use_ipex: + return torch.xpu.mem_get_info("xpu") + else: + index = self.device.index if self.device.index is not None else torch.cuda.current_device() + return torch.cuda.mem_get_info(index) def run(self): if self.disabled: return while True: self.run_flag.wait() - torch.cuda.reset_peak_memory_stats() + from modules import shared + if shared.cmd_opts.use_ipex: + torch.xpu.reset_peak_memory_stats() + else: + torch.cuda.reset_peak_memory_stats() self.data.clear() if self.opts.memmon_poll_rate <= 0: self.run_flag.clear() @@ -54,12 +72,19 @@ class MemUsageMonitor(threading.Thread): for k, v in self.read().items(): print(k, -(v // -(1024 ** 2))) print(self, 'raw torch memory stats:') - tm = torch.cuda.memory_stats(self.device) + from modules import shared + if shared.cmd_opts.use_ipex: + tm = torch.xpu.memory_stats("xpu") + else: + tm = torch.cuda.memory_stats(self.device) for k, v in tm.items(): if 'bytes' not in k: continue print('\t' if 'peak' in k else '', k, -(v // -(1024 ** 2))) - print(torch.cuda.memory_summary()) + if shared.cmd_opts.use_ipex: + print(torch.xpu.memory_summary()) + else: + print(torch.cuda.memory_summary()) def monitor(self): self.run_flag.set() @@ -70,7 +95,11 @@ class MemUsageMonitor(threading.Thread): self.data["free"] = free self.data["total"] = total - torch_stats = torch.cuda.memory_stats(self.device) + from modules import shared + if shared.cmd_opts.use_ipex: + torch_stats = torch.xpu.memory_stats("xpu") + else: + torch_stats = torch.cuda.memory_stats(self.device) self.data["active"] = torch_stats["active.all.current"] self.data["active_peak"] = torch_stats["active_bytes.all.peak"] self.data["reserved"] = torch_stats["reserved_bytes.all.current"] diff --git a/modules/processing.py b/modules/processing.py index e793f12a3..293a8d606 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -55,7 +55,17 @@ def memory_stats(): except Exception as e: mem.update({ 'ram': e }) try: - if torch.cuda.is_available(): + from modules import shared + if shared.cmd_opts.use_ipex: + s = torch.xpu.mem_get_info() + gpu = { 'used': gb(s[1] - s[0]), 'total': gb(s[1]) } + s = dict(torch.xpu.memory_stats("xpu")) + mem.update({ + 'gpu': gpu, + 'retries': s['num_alloc_retries'], + 'oom': s['num_ooms'] + }) + elif torch.cuda.is_available(): s = torch.cuda.mem_get_info() gpu = { 'used': gb(s[1] - s[0]), 'total': gb(s[1]) } s = dict(torch.cuda.memory_stats(shared.device)) diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py index 12ee9f956..5168b4b7a 100644 --- a/modules/sd_hijack_optimizations.py +++ b/modules/sd_hijack_optimizations.py @@ -22,7 +22,15 @@ if shared.opts.cross_attention_optimization == "xFormers": def get_available_vram(): - if shared.device.type == 'cuda': + if shared.cmd_opts.use_ipex: + stats = torch.xpu.memory_stats("xpu") + mem_active = stats['active_bytes.all.current'] + mem_reserved = stats['reserved_bytes.all.current'] + mem_free_xpu, _ = torch.xpu.mem_get_info("xpu") + mem_free_torch = mem_reserved - mem_active + mem_free_total = mem_free_xpu + mem_free_torch + return mem_free_total + elif shared.device.type == 'cuda': stats = torch.cuda.memory_stats(shared.device) mem_active = stats['active_bytes.all.current'] mem_reserved = stats['reserved_bytes.all.current'] @@ -189,14 +197,24 @@ def einsum_op_tensor_mem(q, k, v, max_tensor_mb): return einsum_op_slice_1(q, k, v, max(q.shape[1] // div, 1)) def einsum_op_cuda(q, k, v): - stats = torch.cuda.memory_stats(q.device) - mem_active = stats['active_bytes.all.current'] - mem_reserved = stats['reserved_bytes.all.current'] - mem_free_cuda, _ = torch.cuda.mem_get_info(q.device) - mem_free_torch = mem_reserved - mem_active - mem_free_total = mem_free_cuda + mem_free_torch - # Divide factor of safety as there's copying and fragmentation - return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20)) + if shared.cmd_opts.use_ipex: + stats = torch.xpu.memory_stats("xpu") + mem_active = stats['active_bytes.all.current'] + mem_reserved = stats['reserved_bytes.all.current'] + mem_free_xpu, _ = torch.xpu.mem_get_info("xpu") + mem_free_torch = mem_reserved - mem_active + mem_free_total = mem_free_xpu + mem_free_torch + # Divide factor of safety as there's copying and fragmentation + return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20)) + else: + stats = torch.cuda.memory_stats(q.device) + mem_active = stats['active_bytes.all.current'] + mem_reserved = stats['reserved_bytes.all.current'] + mem_free_cuda, _ = torch.cuda.mem_get_info(q.device) + mem_free_torch = mem_reserved - mem_active + mem_free_total = mem_free_cuda + mem_free_torch + # Divide factor of safety as there's copying and fragmentation + return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20)) def einsum_op_dml(q, k, v): mem_total, mem_active = torch.dml.memory_stats(q.device) @@ -204,6 +222,9 @@ def einsum_op_dml(q, k, v): return einsum_op_tensor_mem(q, k, v, (mem_reserved - mem_active) if mem_reserved > mem_active else 1) def einsum_op(q, k, v): + if shared.cmd_opts.use_ipex: + return einsum_op_cuda(q, k, v) + if q.device.type == 'cuda': return einsum_op_cuda(q, k, v) @@ -397,8 +418,12 @@ def scaled_dot_product_attention_forward(self, x, context=None, mask=None): return hidden_states def scaled_dot_product_no_mem_attention_forward(self, x, context=None, mask=None): - with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False): - return scaled_dot_product_attention_forward(self, x, context, mask) + if shared.cmd_opts.use_ipex: + with torch.backends.xpu.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False): + return scaled_dot_product_attention_forward(self, x, context, mask) + else: + with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False): + return scaled_dot_product_attention_forward(self, x, context, mask) def cross_attention_attnblock_forward(self, x): h_ = x @@ -502,8 +527,12 @@ def sdp_attnblock_forward(self, x): return x + out def sdp_no_mem_attnblock_forward(self, x): - with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False): - return sdp_attnblock_forward(self, x) + if shared.cmd_opts.use_ipex: + with torch.backends.xpu.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False): + return sdp_attnblock_forward(self, x) + else: + with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False): + return sdp_attnblock_forward(self, x) def sub_quad_attnblock_forward(self, x): h_ = x diff --git a/modules/sd_hijack_unet.py b/modules/sd_hijack_unet.py index 158582632..7ff553ae3 100644 --- a/modules/sd_hijack_unet.py +++ b/modules/sd_hijack_unet.py @@ -3,6 +3,7 @@ from packaging import version from modules import devices from modules.sd_hijack_utils import CondFunc +from modules import shared class TorchHijackForUnet: @@ -67,7 +68,7 @@ def hijack_ddpm_edit(): unet_needs_upcast = lambda *args, **kwargs: devices.unet_needs_upcast CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.apply_model', apply_model, unet_needs_upcast) CondFunc('ldm.modules.diffusionmodules.openaimodel.timestep_embedding', lambda orig_func, timesteps, *args, **kwargs: orig_func(timesteps, *args, **kwargs).to(torch.float32 if timesteps.dtype == torch.int64 else devices.dtype_unet), unet_needs_upcast) -if version.parse(torch.__version__) <= version.parse("1.13.2") or torch.cuda.is_available(): +if version.parse(torch.__version__) <= version.parse("1.13.2") or torch.cuda.is_available() or shared.cmd_opts.use_ipex: CondFunc('ldm.modules.diffusionmodules.util.GroupNorm32.forward', lambda orig_func, self, *args, **kwargs: orig_func(self.float(), *args, **kwargs), unet_needs_upcast) CondFunc('ldm.modules.attention.GEGLU.forward', lambda orig_func, self, x: orig_func(self.float(), x.float()).to(devices.dtype_unet), unet_needs_upcast) CondFunc('open_clip.transformer.ResidualAttentionBlock.__init__', lambda orig_func, *args, **kwargs: kwargs.update({'act_layer': GELUHijack}) and False or orig_func(*args, **kwargs), lambda _, *args, **kwargs: kwargs.get('act_layer') is None or kwargs['act_layer'] == torch.nn.GELU) diff --git a/modules/sd_models.py b/modules/sd_models.py index a2ef7a012..49de37097 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -533,7 +533,6 @@ def unload_model_weights(sd_model=None, _info=None): sd_model = None gc.collect() devices.torch_gc() - torch.cuda.empty_cache() print(f"Unloaded weights {timer.summary()}") return sd_model diff --git a/modules/shared.py b/modules/shared.py index f3ebbce55..aa2ede29e 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -238,7 +238,7 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), { "comma_padding_backtrack": OptionInfo(20, "Increase coherency by padding from the last comma within n tokens when using more than 75 tokens", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }), "CLIP_stop_at_last_layers": OptionInfo(1, "Clip skip", gr.Slider, {"minimum": 1, "maximum": 12, "step": 1, "visible": False}), "upcast_attn": OptionInfo(False, "Upcast cross attention layer to float32"), - "cross_attention_optimization": OptionInfo("Sub-quadratic" if is_device_dml else "Scaled-Dot-Product", "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }), + "cross_attention_optimization": OptionInfo("Sub-quadratic" if is_device_dml else "Split attention" if cmd_opts.use_ipex else "Scaled-Dot-Product", "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }), "cross_attention_options": OptionInfo([], "Cross-attention advanced options", gr.CheckboxGroup, lambda: {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention']}), "sub_quad_q_chunk_size": OptionInfo(512, "Sub-quadratic cross-attention query chunk size for the layer optimization to use", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8}), "sub_quad_kv_chunk_size": OptionInfo(512, "Sub-quadratic cross-attentionkv chunk size for the sub-quadratic cross-attention layer optimization to use", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8}), @@ -318,7 +318,7 @@ options_templates.update(options_section(('cuda', "CUDA Settings"), { "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" else "FP16", "Device precision type", gr.Radio, lambda: {"choices": ["FP32", "FP16", "BF16"]}), "no_half": OptionInfo(True if is_device_dml else False, "Use full precision for model (--no-half)", None, None, lambda: print("Warning: Most of DirectML devices do not fully support half mode. Recommend to use full precision to model.") if is_device_dml else None), "no_half_vae": OptionInfo(True if is_device_dml else False, "Use full precision for VAE (--no-half-vae)"), - "upcast_sampling": OptionInfo(True if sys.platform == "darwin" else False, "Enable upcast sampling. Usually produces similar results to --no-half with better performance while using less memory"), + "upcast_sampling": OptionInfo(True if sys.platform == "darwin" or cmd_opts.use_ipex else False, "Enable upcast sampling. Usually produces similar results to --no-half with better performance while using less memory"), "disable_nan_check": OptionInfo(True, "Do not check if produced images/latent spaces have NaN values"), "rollback_vae": OptionInfo(False, "Attempt to roll back VAE when produced NaN values, requires NaN check (experimental)"), "opt_channelslast": OptionInfo(False, "Use channels last as torch memory format "), diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index 2180d7f32..36a1e1e17 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -434,7 +434,11 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st else: print("No saved optimizer exists in checkpoint") - scaler = torch.cuda.amp.GradScaler() + from modules import shared + if shared.cmd_opts.use_ipex: + scaler = torch.xpu.amp.GradScaler() + else: + scaler = torch.cuda.amp.GradScaler() batch_size = ds.batch_size gradient_step = ds.gradient_step diff --git a/setup.py b/setup.py index cfaa1aaa0..59c200161 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ def setup_logging(clean=False): # check if package is installed def installed(package, friendly: str = None): import pkg_resources + from modules import shared ok = True try: if friendly: @@ -76,6 +77,8 @@ def installed(package, friendly: str = None): ok = ok and spec is not None if ok: version = pkg_resources.get_distribution(p[0]).version + if shared.cmd_opts.use_ipex and p[0] == "pytorch_lightning": + p[1] = "1.8.6" log.debug(f"Package version found: {p[0]} {version}") if len(p) > 1: ok = ok and version == p[1] @@ -91,6 +94,9 @@ def installed(package, friendly: str = None): # install package using pip if not already installed def install(package, friendly: str = None, ignore: bool = False): + from modules import shared + if shared.cmd_opts.use_ipex and package == "pytorch_lightning==1.9.4": + package = "pytorch_lightning==1.8.6" def pip(arg: str): arg = arg.replace('>=', '==') log.info(f'Installing package: {arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace(" ", " ").strip()}') @@ -188,6 +194,7 @@ def check_python(): # check torch version def check_torch(): + from modules import shared if shutil.which('nvidia-smi') is not None or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe')): log.info('nVidia toolkit detected') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchaudio torchvision --index-url https://download.pytorch.org/whl/cu118') @@ -197,6 +204,11 @@ def check_torch(): os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', '10.3.0') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.4.2') xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') + elif shutil.which('sycl-ls') is not None or os.path.exists('/opt/intel/oneapi'): + shared.cmd_opts.use_ipex = True + log.info('Intel toolkit detected') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==1.13.0a0+git6c9b55e torchvision==0.14.1a0 intel_extension_for_pytorch==1.13.120+xpu --index-url https://developer.intel.com/ipex-whl-stable-xpu') + xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') else: machine = platform.machine() if 'arm' not in machine and 'aarch' not in machine and not args.nodirectml: # torch-directml is available on AMD64 @@ -212,7 +224,11 @@ def check_torch(): try: import torch log.info(f'Torch {torch.__version__}') - if torch.cuda.is_available(): + if shared.cmd_opts.use_ipex: + import intel_extension_for_pytorch as ipex + log.info(f'Torch backend: Intel OneAPI {torch.__version__}') + log.info(f'Torch detected GPU: {torch.xpu.get_device_name("xpu")} VRAM {round(torch.xpu.get_device_properties("xpu").total_memory / 1024 / 1024)}') + elif torch.cuda.is_available(): if torch.version.cuda: log.info(f'Torch backend: nVidia CUDA {torch.version.cuda} cuDNN {torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else "N/A"}') elif torch.version.hip: From 5c76087b9d4e90edb7f79631c3ef1cc3e627327d Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 30 Apr 2023 15:30:15 +0300 Subject: [PATCH 2/6] Revert force cross_attention_optimization --- modules/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shared.py b/modules/shared.py index aa2ede29e..12cb05b05 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -238,7 +238,7 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), { "comma_padding_backtrack": OptionInfo(20, "Increase coherency by padding from the last comma within n tokens when using more than 75 tokens", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }), "CLIP_stop_at_last_layers": OptionInfo(1, "Clip skip", gr.Slider, {"minimum": 1, "maximum": 12, "step": 1, "visible": False}), "upcast_attn": OptionInfo(False, "Upcast cross attention layer to float32"), - "cross_attention_optimization": OptionInfo("Sub-quadratic" if is_device_dml else "Split attention" if cmd_opts.use_ipex else "Scaled-Dot-Product", "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }), + "cross_attention_optimization": OptionInfo("Sub-quadratic" if is_device_dml else "Scaled-Dot-Product", "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }), "cross_attention_options": OptionInfo([], "Cross-attention advanced options", gr.CheckboxGroup, lambda: {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention']}), "sub_quad_q_chunk_size": OptionInfo(512, "Sub-quadratic cross-attention query chunk size for the layer optimization to use", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8}), "sub_quad_kv_chunk_size": OptionInfo(512, "Sub-quadratic cross-attentionkv chunk size for the sub-quadratic cross-attention layer optimization to use", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8}), From a720a670e826715790d095d26a3829126f6b7811 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 30 Apr 2023 16:01:17 +0300 Subject: [PATCH 3/6] More patches and less import shared --- cli/modules/bench.py | 7 ++- cli/modules/interrogate-offline.py | 18 ++++++- cli/modules/lora-extract.py | 16 +++++- cli/modules/lora-latents.py | 11 ++++- cli/modules/util.py | 21 +++++++- cli/random/dynamotest.py | 49 ++++++++++++++----- cli/train-lora.py | 13 ++++- cli/train/latents.py | 11 ++++- .../multidiffusion-upscaler-for-automatic1111 | 2 +- extensions-builtin/sd-webui-controlnet | 2 +- modules/devices.py | 17 +++---- modules/lora | 2 +- 12 files changed, 132 insertions(+), 37 deletions(-) diff --git a/cli/modules/bench.py b/cli/modules/bench.py index 094b73f63..18791bfc9 100755 --- a/cli/modules/bench.py +++ b/cli/modules/bench.py @@ -10,7 +10,7 @@ import time from PIL import Image import sdapi from util import Map, log - +from modules import shared options = Map({ 'restore_faces': False, @@ -56,7 +56,10 @@ async def txt2img(): def memstats(): mem = sdapi.getsync('/sdapi/v1/memory') cpu = mem.get('ram', 'unavailable') - gpu = mem.get('cuda', 'unavailable') + if shared.cmd_opts.use_ipex: + gpu = mem.get('xpu', 'unavailable') + else: + gpu = mem.get('cuda', 'unavailable') if 'active' in gpu: gpu['session'] = gpu.pop('active') if 'reserved' in gpu: diff --git a/cli/modules/interrogate-offline.py b/cli/modules/interrogate-offline.py index 6d9ae56fa..c2623cda6 100755 --- a/cli/modules/interrogate-offline.py +++ b/cli/modules/interrogate-offline.py @@ -6,6 +6,12 @@ import json import time import argparse import torch +from modules import shared +try: + import intel_extension_for_pytorch as ipex +except: + if shared.cmd_opts.use_ipex: + print("Failed to import IPEX") import filetype from PIL import Image import transformers @@ -19,7 +25,10 @@ model = None processor = None extractor = None dtype = torch.float32 -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +if shared.cmd_opts.use_ipex: + device = torch.device('xpu') +else: + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') options = Map({ 'input': '', @@ -129,7 +138,12 @@ def unload_model(): del extractor extractor = None gc.collect() - if torch.cuda.is_available(): + if shared.cmd_opts.use_ipex: + with torch.no_grad(): + torch.xpu.empty_cache() + with torch.xpu.device('xpu'): + torch.xpu.empty_cache() + elif torch.cuda.is_available(): with torch.no_grad(): torch.cuda.empty_cache() with torch.cuda.device('cuda'): diff --git a/cli/modules/lora-extract.py b/cli/modules/lora-extract.py index 102728308..9a781789a 100755 --- a/cli/modules/lora-extract.py +++ b/cli/modules/lora-extract.py @@ -10,6 +10,12 @@ import sys import time import argparse import torch +from modules import shared +try: + import intel_extension_for_pytorch as ipex +except: + if shared.cmd_opts.use_ipex: + print("Failed to import IPEX") import transformers from tqdm import tqdm from util import log @@ -20,7 +26,10 @@ import networks.lora as lora def svd(args): # pylint: disable=redefined-outer-name - device = 'cuda' if torch.cuda.is_available() and args.device == 'cuda' else 'cpu' + if shared.cmd_opts.use_ipex: + device = torch.device('xpu') + else: + device = 'cuda' if torch.cuda.is_available() and args.device == 'cuda' else 'cpu' transformers.logging.set_verbosity_error() CLAMP_QUANTILE = 0.99 MIN_DIFF = 1e-6 @@ -38,7 +47,10 @@ def svd(args): # pylint: disable=redefined-outer-name log.info({ 'loading model': args.tuned }) text_encoder_t, _, unet_t = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.tuned) with torch.no_grad(): - torch.cuda.empty_cache() + if shared.cmd_opts.use_ipex: + torch.xpu.empty_cache() + else: + torch.cuda.empty_cache() # create LoRA network to extract weights: Use dim (rank) as alpha lora_network_o = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_o, unet_o) lora_network_t = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_t, unet_t) diff --git a/cli/modules/lora-latents.py b/cli/modules/lora-latents.py index d556d596b..7e701df12 100755 --- a/cli/modules/lora-latents.py +++ b/cli/modules/lora-latents.py @@ -10,6 +10,12 @@ import warnings import cv2 import numpy as np import torch +from modules import shared +try: + import intel_extension_for_pytorch as ipex +except: + if shared.cmd_opts.use_ipex: + print("Failed to import IPEX") from PIL import Image from torchvision import transforms from tqdm import tqdm @@ -20,7 +26,10 @@ import library.model_util as model_util import library.train_util as train_util warnings.filterwarnings('ignore') -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +if shared.cmd_opts.use_ipex: + device = torch.device('xpu') +else: + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') options = Map({ 'batch': 1, 'input': '', diff --git a/cli/modules/util.py b/cli/modules/util.py index 479b77233..d48887961 100755 --- a/cli/modules/util.py +++ b/cli/modules/util.py @@ -44,7 +44,26 @@ def get_memory(): mem.update({ 'ram': e }) try: import torch - if torch.cuda.is_available(): + from modules import shared + if shared.cmd_opts.use_ipex: + import intel_extension_for_pytorch as ipex + s = torch.xpu.mem_get_info() + gpu = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) } + s = dict(torch.xpu.memory_stats('xpu')) + allocated = { 'current': gb(s['allocated_bytes.all.current']), 'peak': gb(s['allocated_bytes.all.peak']) } + reserved = { 'current': gb(s['reserved_bytes.all.current']), 'peak': gb(s['reserved_bytes.all.peak']) } + active = { 'current': gb(s['active_bytes.all.current']), 'peak': gb(s['active_bytes.all.peak']) } + inactive = { 'current': gb(s['inactive_split_bytes.all.current']), 'peak': gb(s['inactive_split_bytes.all.peak']) } + warnings = { 'retries': s['num_alloc_retries'], 'oom': s['num_ooms'] } + mem.update({ + 'gpu': gpu, + 'gpu-active': active, + 'gpu-allocated': allocated, + 'gpu-reserved': reserved, + 'gpu-inactive': inactive, + 'events': warnings, + }) + elif torch.cuda.is_available(): s = torch.cuda.mem_get_info() gpu = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) } s = dict(torch.cuda.memory_stats('cuda')) diff --git a/cli/random/dynamotest.py b/cli/random/dynamotest.py index 82b1143c6..556ae96a8 100755 --- a/cli/random/dynamotest.py +++ b/cli/random/dynamotest.py @@ -7,9 +7,14 @@ import warnings import numpy as np import torch +from modules import shared +try: + import intel_extension_for_pytorch as ipex +except: + if shared.cmd_opts.use_ipex: + print("Failed to import IPEX") from torchvision.models import resnet18 - print('torch:', torch.__version__) try: import torch._dynamo as dynamo # must be imported explicitly or namespace is not found @@ -24,24 +29,42 @@ warnings.filterwarnings('ignore', category=UserWarning) # disable those for now def timed(fn): # returns the result of running `fn()` and the time it took for `fn()` to run in ms using CUDA events - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - start.record() - result = fn() - end.record() - torch.cuda.synchronize() - return result, start.elapsed_time(end) + if shared.cmd_opts.use_ipex: + start = torch.xpu.Event(enable_timing=True) + end = torch.xpu.Event(enable_timing=True) + start.record() + result = fn() + end.record() + torch.xpu.synchronize() + return result, start.elapsed_time(end) + else: + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + result = fn() + end.record() + torch.cuda.synchronize() + return result, start.elapsed_time(end) def generate_data(b): - return ( - torch.randn(b, 3, 128, 128).to(torch.float32).cuda(), - torch.randint(1000, (b,)).cuda(), - ) + if shared.cmd_opts.use_ipex: + return ( + torch.randn(b, 3, 128, 128).to(torch.float32).xpu(), + torch.randint(1000, (b,)).xpu(), + ) + else: + return ( + torch.randn(b, 3, 128, 128).to(torch.float32).cuda(), + torch.randint(1000, (b,)).cuda(), + ) def init_model(): - return resnet18().to(torch.float32).cuda() + if shared.cmd_opts.use_ipex: + return resnet18().to(torch.float32).xpu() + else: + return resnet18().to(torch.float32).cuda() def eval(mod, inp): diff --git a/cli/train-lora.py b/cli/train-lora.py index 6f82063a6..f1e295a35 100755 --- a/cli/train-lora.py +++ b/cli/train-lora.py @@ -23,6 +23,12 @@ import shutil import argparse import tempfile import torch +from modules import shared +try: + import intel_extension_for_pytorch as ipex +except: + if shared.cmd_opts.use_ipex: + print("Failed to import IPEX") import logging import importlib import transformers @@ -117,7 +123,12 @@ options = Map({ def mem_stats(): gc.collect() - if torch.cuda.is_available(): + if shared.cmd_opts.use_ipex: + with torch.no_grad(): + torch.xpu.empty_cache() + with torch.xpu.device('xpu'): + torch.cuda.empty_cache() + elif torch.cuda.is_available(): with torch.no_grad(): torch.cuda.empty_cache() with torch.cuda.device('cuda'): diff --git a/cli/train/latents.py b/cli/train/latents.py index 94249b18a..715f92aba 100755 --- a/cli/train/latents.py +++ b/cli/train/latents.py @@ -10,6 +10,12 @@ import warnings import cv2 import numpy as np import torch +from modules import shared +try: + import intel_extension_for_pytorch as ipex +except: + if shared.cmd_opts.use_ipex: + print("Failed to import IPEX") from PIL import Image from torchvision import transforms from tqdm import tqdm @@ -28,7 +34,10 @@ import library.model_util as model_util import library.train_util as train_util warnings.filterwarnings('ignore') -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +if shared.cmd_opts.use_ipex: + device = torch.device('xpu') +else: + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') options = Map({ 'batch': 1, 'input': '', diff --git a/extensions-builtin/multidiffusion-upscaler-for-automatic1111 b/extensions-builtin/multidiffusion-upscaler-for-automatic1111 index 6931b89cb..860f8a405 160000 --- a/extensions-builtin/multidiffusion-upscaler-for-automatic1111 +++ b/extensions-builtin/multidiffusion-upscaler-for-automatic1111 @@ -1 +1 @@ -Subproject commit 6931b89cb4507c7dc8fa81ac36c2c19d0691c44e +Subproject commit 860f8a405193bcd992e21d82e43fa18137bc4923 diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 4d4b1f8c0..09d1fcbf4 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 4d4b1f8c00a0355d1517465ac3c0e801d5a2d194 +Subproject commit 09d1fcbf4dc715bca7547496f850801f95f732a6 diff --git a/modules/devices.py b/modules/devices.py index 3606597d3..1ab082f33 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -1,6 +1,12 @@ import sys import contextlib import torch +from modules import shared +try: + import intel_extension_for_pytorch as ipex +except: + if shared.cmd_opts.use_ipex: + print("Failed to import IPEX") if sys.platform == "darwin": from modules import mac_specific @@ -21,25 +27,21 @@ def extract_device_id(args, name): def get_cuda_device_string(): - from modules import shared if shared.cmd_opts.use_ipex: return "xpu" else: - from modules import shared if shared.cmd_opts.device_id is not None: return f"cuda:{shared.cmd_opts.device_id}" return "cuda" def get_dml_device_string(): - from modules import shared if shared.cmd_opts.device_id is not None: return f"privateuseone:{shared.cmd_opts.device_id}" return "privateuseone:0" def get_optimal_device_name(): - from modules import shared if shared.cmd_opts.use_ipex: return "xpu" elif torch.cuda.is_available(): @@ -61,14 +63,12 @@ def get_optimal_device(): def get_device_for(task): - from modules import shared if task in shared.cmd_opts.use_cpu: return cpu return get_optimal_device() def torch_gc(): - from modules import shared if shared.cmd_opts.use_ipex: with torch.xpu.device("xpu"): torch.xpu.empty_cache() @@ -79,7 +79,6 @@ def torch_gc(): def set_cuda_params(): - from modules import shared if torch.cuda.is_available(): try: torch.backends.cuda.matmul.allow_tf32 = shared.opts.cuda_allow_tf32 @@ -143,12 +142,10 @@ def randn_without_seed(shape): def autocast(disable=False): - from modules import shared if disable: return contextlib.nullcontext() if dtype == torch.float32 or shared.cmd_opts.precision == "Full": return contextlib.nullcontext() - from modules import shared if shared.cmd_opts.use_ipex: return torch.xpu.amp.autocast(enabled=True, dtype=dtype, cache_enabled=False) else: @@ -156,7 +153,6 @@ def autocast(disable=False): def without_autocast(disable=False): - from modules import shared if shared.cmd_opts.use_ipex: return torch.autocast("xpu", enabled=False) if torch.is_autocast_enabled() and not disable else contextlib.nullcontext() else: @@ -168,7 +164,6 @@ class NansException(Exception): def test_for_nans(x, where): - from modules import shared if shared.opts.disable_nan_check: return if not torch.all(torch.isnan(x)).item(): diff --git a/modules/lora b/modules/lora index d52c524fc..bc803e01c 160000 --- a/modules/lora +++ b/modules/lora @@ -1 +1 @@ -Subproject commit d52c524fc2942c053cf37c648188502a3a26df1b +Subproject commit bc803e01c7028471efc8db5bc9aa183fde06080c From de8d0bef9f64cab4e83845e2055d671181f5b4a6 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 30 Apr 2023 18:19:37 +0300 Subject: [PATCH 4/6] More patches and Import IPEX after Torch --- extensions-builtin/sd-webui-controlnet | 2 +- modules/api/api.py | 19 ++++++++++++++++++- modules/codeformer/codeformer_arch.py | 4 ++++ modules/codeformer/vqgan_arch.py | 4 ++++ modules/codeformer_model.py | 7 +++++-- modules/deepbooru.py | 4 ++++ modules/deepbooru_model.py | 4 ++++ modules/devices.py | 3 +-- modules/esrgan_model.py | 4 ++++ modules/esrgan_model_arch.py | 4 ++++ modules/extras.py | 4 ++++ modules/hypernetworks/hypernetwork.py | 17 ++++++++++++++--- modules/interrogate.py | 4 ++++ modules/lowvram.py | 4 ++++ modules/mac_specific.py | 4 ++++ modules/memmon.py | 13 +++++++------ modules/models/diffusion/ddpm_edit.py | 4 ++++ modules/models/diffusion/uni_pc/sampler.py | 4 ++++ modules/models/diffusion/uni_pc/uni_pc.py | 4 ++++ modules/processing.py | 10 ++++++---- modules/prompt_parser.py | 4 ++++ modules/safe.py | 4 ++++ modules/sd_disable_initialization.py | 4 ++++ modules/sd_hijack.py | 4 ++++ modules/sd_hijack_clip.py | 4 ++++ modules/sd_hijack_inpainting.py | 4 ++++ modules/sd_hijack_open_clip.py | 4 ++++ modules/sd_hijack_optimizations.py | 8 ++++++-- modules/sd_hijack_unet.py | 4 ++++ modules/sd_hijack_xlmr.py | 4 ++++ modules/sd_models.py | 4 ++++ modules/sd_models_config.py | 4 ++++ modules/sd_samplers_common.py | 4 ++++ modules/sd_samplers_compvis.py | 4 ++++ modules/sd_samplers_kdiffusion.py | 4 ++++ modules/sd_vae.py | 8 +++++++- modules/sd_vae_approx.py | 4 ++++ modules/sub_quadratic_attention.py | 4 ++++ modules/textual_inversion/dataset.py | 4 ++++ modules/textual_inversion/image_embedding.py | 4 ++++ .../textual_inversion/textual_inversion.py | 5 ++++- modules/xlmr.py | 4 ++++ webui.py | 4 ++++ wiki | 2 +- 44 files changed, 202 insertions(+), 24 deletions(-) diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 09d1fcbf4..d2da774a4 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 09d1fcbf4dc715bca7547496f850801f95f732a6 +Subproject commit d2da774a40ff9c3770e21f71fb516403022fc3f6 diff --git a/modules/api/api.py b/modules/api/api.py index 0717edfaf..fdd26f868 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -573,7 +573,24 @@ class Api: ram = { 'error': f'{err}' } try: import torch - if torch.cuda.is_available(): + if shared.cmd_opts.use_ipex(): + import intel_extension_for_pytorch as ipex + system = { 'free': (torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), 'used': torch.xpu.memory_allocated(), 'total': torch.xpu.get_device_properties("xpu").total_memory } + s = dict(torch.xpu.memory_stats("xpu")) + allocated = { 'current': s['allocated_bytes.all.current'], 'peak': s['allocated_bytes.all.peak'] } + reserved = { 'current': s['reserved_bytes.all.current'], 'peak': s['reserved_bytes.all.peak'] } + active = { 'current': s['active_bytes.all.current'], 'peak': s['active_bytes.all.peak'] } + inactive = { 'current': s['inactive_split_bytes.all.current'], 'peak': s['inactive_split_bytes.all.peak'] } + warnings = { 'retries': s['num_alloc_retries'], 'oom': s['num_ooms'] } + cuda = { + 'system': system, + 'active': active, + 'allocated': allocated, + 'reserved': reserved, + 'inactive': inactive, + 'events': warnings, + } + elif torch.cuda.is_available(): s = torch.cuda.mem_get_info() system = { 'free': s[0], 'used': s[1] - s[0], 'total': s[1] } s = dict(torch.cuda.memory_stats(shared.device)) diff --git a/modules/codeformer/codeformer_arch.py b/modules/codeformer/codeformer_arch.py index 11dcc3ee7..6d7b926fe 100644 --- a/modules/codeformer/codeformer_arch.py +++ b/modules/codeformer/codeformer_arch.py @@ -3,6 +3,10 @@ import math import numpy as np import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from torch import nn, Tensor import torch.nn.functional as F from typing import Optional, List diff --git a/modules/codeformer/vqgan_arch.py b/modules/codeformer/vqgan_arch.py index e72936838..e66bb2a72 100644 --- a/modules/codeformer/vqgan_arch.py +++ b/modules/codeformer/vqgan_arch.py @@ -7,6 +7,10 @@ https://github.com/samb-t/unleashing-transformers/blob/master/models/vqgan.py ''' import numpy as np import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import torch.nn as nn import torch.nn.functional as F import copy diff --git a/modules/codeformer_model.py b/modules/codeformer_model.py index 5217f69db..9d75e823d 100644 --- a/modules/codeformer_model.py +++ b/modules/codeformer_model.py @@ -3,6 +3,10 @@ import sys import cv2 import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import modules.face_restoration from modules import shared, devices, modelloader, errors @@ -103,8 +107,7 @@ def setup_model(dirname): output = self.net(cropped_face_t, w=w if w is not None else shared.opts.code_former_weight, adain=True)[0] restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1)) del output - from modules import shared - if shared.cmd_opts.use_ipex: + if cmd_opts.use_ipex: torch.xpu.empty_cache() else: torch.cuda.empty_cache() diff --git a/modules/deepbooru.py b/modules/deepbooru.py index 1c4554a20..50e400fd8 100644 --- a/modules/deepbooru.py +++ b/modules/deepbooru.py @@ -2,6 +2,10 @@ import os import re import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import numpy as np from modules import modelloader, paths, deepbooru_model, devices, images, shared diff --git a/modules/deepbooru_model.py b/modules/deepbooru_model.py index c2c77cd25..ef53494a2 100644 --- a/modules/deepbooru_model.py +++ b/modules/deepbooru_model.py @@ -1,4 +1,8 @@ import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import torch.nn as nn import torch.nn.functional as F diff --git a/modules/devices.py b/modules/devices.py index 1ab082f33..8be1e3866 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -5,8 +5,7 @@ from modules import shared try: import intel_extension_for_pytorch as ipex except: - if shared.cmd_opts.use_ipex: - print("Failed to import IPEX") + pass if sys.platform == "darwin": from modules import mac_specific diff --git a/modules/esrgan_model.py b/modules/esrgan_model.py index bb4c6619b..769d66f01 100644 --- a/modules/esrgan_model.py +++ b/modules/esrgan_model.py @@ -2,6 +2,10 @@ import os import numpy as np import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from PIL import Image from basicsr.utils.download_util import load_file_from_url diff --git a/modules/esrgan_model_arch.py b/modules/esrgan_model_arch.py index 411d98d38..fc352d0ba 100644 --- a/modules/esrgan_model_arch.py +++ b/modules/esrgan_model_arch.py @@ -2,6 +2,10 @@ import math import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import torch.nn as nn import torch.nn.functional as F diff --git a/modules/extras.py b/modules/extras.py index c0ae9477f..4513f2491 100644 --- a/modules/extras.py +++ b/modules/extras.py @@ -4,6 +4,10 @@ import html import shutil import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import tqdm import gradio as gr import safetensors.torch diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index 4aa5ffcdc..a1caecbe4 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -8,6 +8,10 @@ import inspect import modules.textual_inversion.dataset import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import tqdm from einops import rearrange, repeat from ldm.util import default @@ -591,7 +595,10 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi print("Cannot resume from saved optimizer!") print(e) - scaler = torch.cuda.amp.GradScaler() + if shared.cmd_opts.use_ipex: + scaler = torch.xpu.amp.GradScaler() + else: + scaler = torch.cuda.amp.GradScaler() batch_size = ds.batch_size gradient_step = ds.gradient_step @@ -708,7 +715,9 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi hypernetwork.eval() rng_state = torch.get_rng_state() cuda_rng_state = None - if torch.cuda.is_available(): + if shared.cmd_opts.use_ipex: + cuda_rng_state = torch.xpu.get_rng_state_all() + elif torch.cuda.is_available(): cuda_rng_state = torch.cuda.get_rng_state_all() shared.sd_model.cond_stage_model.to(devices.device) shared.sd_model.first_stage_model.to(devices.device) @@ -745,7 +754,9 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi shared.sd_model.cond_stage_model.to(devices.cpu) shared.sd_model.first_stage_model.to(devices.cpu) torch.set_rng_state(rng_state) - if torch.cuda.is_available(): + if shared.cmd_opts.use_ipex: + torch.xpu.set_rng_state_all(cuda_rng_state) + elif torch.cuda.is_available(): torch.cuda.set_rng_state_all(cuda_rng_state) hypernetwork.train() if image is not None: diff --git a/modules/interrogate.py b/modules/interrogate.py index 6afbde570..93bb08f20 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -5,6 +5,10 @@ from pathlib import Path import re import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import torch.hub from torchvision import transforms diff --git a/modules/lowvram.py b/modules/lowvram.py index e254cc131..7dba01593 100644 --- a/modules/lowvram.py +++ b/modules/lowvram.py @@ -1,4 +1,8 @@ import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from modules import devices module_in_gpu = None diff --git a/modules/mac_specific.py b/modules/mac_specific.py index c8a534d0e..2455800d5 100644 --- a/modules/mac_specific.py +++ b/modules/mac_specific.py @@ -1,4 +1,8 @@ import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import platform from modules.sd_hijack_utils import CondFunc from packaging import version diff --git a/modules/memmon.py b/modules/memmon.py index 4ceb29a37..3abc70ac3 100644 --- a/modules/memmon.py +++ b/modules/memmon.py @@ -2,6 +2,12 @@ import threading import time from collections import defaultdict import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass + +from modules import shared class MemUsageMonitor(threading.Thread): @@ -19,7 +25,6 @@ class MemUsageMonitor(threading.Thread): self.daemon = True self.run_flag = threading.Event() self.data = defaultdict(int) - from modules import shared if not torch.cuda.is_available() or not shared.cmd_opts.use_ipex: self.disabled = True else: @@ -40,9 +45,8 @@ class MemUsageMonitor(threading.Thread): self.disabled = True def cuda_mem_get_info(self): - from modules import shared if shared.cmd_opts.use_ipex: - return torch.xpu.mem_get_info("xpu") + return [(torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), torch.xpu.get_device_properties("xpu").total_memory] else: index = self.device.index if self.device.index is not None else torch.cuda.current_device() return torch.cuda.mem_get_info(index) @@ -52,7 +56,6 @@ class MemUsageMonitor(threading.Thread): return while True: self.run_flag.wait() - from modules import shared if shared.cmd_opts.use_ipex: torch.xpu.reset_peak_memory_stats() else: @@ -72,7 +75,6 @@ class MemUsageMonitor(threading.Thread): for k, v in self.read().items(): print(k, -(v // -(1024 ** 2))) print(self, 'raw torch memory stats:') - from modules import shared if shared.cmd_opts.use_ipex: tm = torch.xpu.memory_stats("xpu") else: @@ -95,7 +97,6 @@ class MemUsageMonitor(threading.Thread): self.data["free"] = free self.data["total"] = total - from modules import shared if shared.cmd_opts.use_ipex: torch_stats = torch.xpu.memory_stats("xpu") else: diff --git a/modules/models/diffusion/ddpm_edit.py b/modules/models/diffusion/ddpm_edit.py index f3d49c44c..846a74fc4 100644 --- a/modules/models/diffusion/ddpm_edit.py +++ b/modules/models/diffusion/ddpm_edit.py @@ -10,6 +10,10 @@ https://github.com/CompVis/taming-transformers # See more details in LICENSE. import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import torch.nn as nn import numpy as np import pytorch_lightning as pl diff --git a/modules/models/diffusion/uni_pc/sampler.py b/modules/models/diffusion/uni_pc/sampler.py index 3100522ab..6dd7c7fd8 100644 --- a/modules/models/diffusion/uni_pc/sampler.py +++ b/modules/models/diffusion/uni_pc/sampler.py @@ -2,6 +2,10 @@ import numpy as np import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from .uni_pc import NoiseScheduleVP, model_wrapper, UniPC from modules import shared, devices diff --git a/modules/models/diffusion/uni_pc/uni_pc.py b/modules/models/diffusion/uni_pc/uni_pc.py index 61ee39522..895fc58c3 100644 --- a/modules/models/diffusion/uni_pc/uni_pc.py +++ b/modules/models/diffusion/uni_pc/uni_pc.py @@ -1,4 +1,8 @@ import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import torch.nn.functional as F import math import time diff --git a/modules/processing.py b/modules/processing.py index 293a8d606..36737fdbe 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -8,6 +8,10 @@ from typing import Any, Dict, List import psutil import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import numpy as np from PIL import Image, ImageFilter, ImageOps import cv2 @@ -55,10 +59,8 @@ def memory_stats(): except Exception as e: mem.update({ 'ram': e }) try: - from modules import shared - if shared.cmd_opts.use_ipex: - s = torch.xpu.mem_get_info() - gpu = { 'used': gb(s[1] - s[0]), 'total': gb(s[1]) } + if cmd_opts.use_ipex: + gpu = { 'used': gb(torch.xpu.memory_allocated()), 'total': gb(torch.xpu.get_device_properties("xpu").total_memory) } s = dict(torch.xpu.memory_stats("xpu")) mem.update({ 'gpu': gpu, diff --git a/modules/prompt_parser.py b/modules/prompt_parser.py index 7006f2822..6722d9f80 100644 --- a/modules/prompt_parser.py +++ b/modules/prompt_parser.py @@ -368,3 +368,7 @@ if __name__ == "__main__": doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE) else: import torch # doctest faster + try: + import intel_extension_for_pytorch as ipex + except: + pass diff --git a/modules/safe.py b/modules/safe.py index 9a1133ddc..dd463ccdd 100644 --- a/modules/safe.py +++ b/modules/safe.py @@ -6,6 +6,10 @@ import zipfile import re import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import numpy import _codecs diff --git a/modules/sd_disable_initialization.py b/modules/sd_disable_initialization.py index c4a09d15d..5cc5e4e7a 100644 --- a/modules/sd_disable_initialization.py +++ b/modules/sd_disable_initialization.py @@ -1,6 +1,10 @@ import ldm.modules.encoders.modules import open_clip import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import transformers.utils.hub diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index f817b7afd..6d2bb3b0f 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -1,6 +1,10 @@ from types import MethodType from rich import print # pylint: disable=redefined-builtin import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from torch.nn.functional import silu import ldm.modules.attention import ldm.modules.diffusionmodules.model diff --git a/modules/sd_hijack_clip.py b/modules/sd_hijack_clip.py index 945f7732d..cf4abf84f 100644 --- a/modules/sd_hijack_clip.py +++ b/modules/sd_hijack_clip.py @@ -2,6 +2,10 @@ import math from collections import namedtuple import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from modules import prompt_parser, devices, sd_hijack from modules.shared import opts diff --git a/modules/sd_hijack_inpainting.py b/modules/sd_hijack_inpainting.py index 4b23c132d..1a9ea9b4c 100644 --- a/modules/sd_hijack_inpainting.py +++ b/modules/sd_hijack_inpainting.py @@ -1,4 +1,8 @@ import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import ldm.models.diffusion.ddpm import ldm.models.diffusion.ddim diff --git a/modules/sd_hijack_open_clip.py b/modules/sd_hijack_open_clip.py index f76fc1f3b..c0c204a82 100644 --- a/modules/sd_hijack_open_clip.py +++ b/modules/sd_hijack_open_clip.py @@ -1,5 +1,9 @@ import open_clip.tokenizer import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from modules import sd_hijack_clip, devices diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py index 5168b4b7a..3887e238d 100644 --- a/modules/sd_hijack_optimizations.py +++ b/modules/sd_hijack_optimizations.py @@ -2,6 +2,10 @@ import math import psutil import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from torch import einsum from ldm.util import default @@ -26,7 +30,7 @@ def get_available_vram(): stats = torch.xpu.memory_stats("xpu") mem_active = stats['active_bytes.all.current'] mem_reserved = stats['reserved_bytes.all.current'] - mem_free_xpu, _ = torch.xpu.mem_get_info("xpu") + mem_free_xpu, _ = [(torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), torch.xpu.get_device_properties("xpu").total_memory] mem_free_torch = mem_reserved - mem_active mem_free_total = mem_free_xpu + mem_free_torch return mem_free_total @@ -201,7 +205,7 @@ def einsum_op_cuda(q, k, v): stats = torch.xpu.memory_stats("xpu") mem_active = stats['active_bytes.all.current'] mem_reserved = stats['reserved_bytes.all.current'] - mem_free_xpu, _ = torch.xpu.mem_get_info("xpu") + mem_free_xpu, _ = [(torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), torch.xpu.get_device_properties("xpu").total_memory] mem_free_torch = mem_reserved - mem_active mem_free_total = mem_free_xpu + mem_free_torch # Divide factor of safety as there's copying and fragmentation diff --git a/modules/sd_hijack_unet.py b/modules/sd_hijack_unet.py index 7ff553ae3..ce6ac1306 100644 --- a/modules/sd_hijack_unet.py +++ b/modules/sd_hijack_unet.py @@ -1,4 +1,8 @@ import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from packaging import version from modules import devices diff --git a/modules/sd_hijack_xlmr.py b/modules/sd_hijack_xlmr.py index 28528329b..a9cb9454c 100644 --- a/modules/sd_hijack_xlmr.py +++ b/modules/sd_hijack_xlmr.py @@ -1,4 +1,8 @@ import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from modules import sd_hijack_clip, devices diff --git a/modules/sd_models.py b/modules/sd_models.py index 49de37097..2e2b5f0a2 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -8,6 +8,10 @@ from os import mkdir from urllib import request from rich import print, progress # pylint: disable=redefined-builtin import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import safetensors.torch from omegaconf import OmegaConf import tomesd diff --git a/modules/sd_models_config.py b/modules/sd_models_config.py index a9c515b14..5bc3799a0 100644 --- a/modules/sd_models_config.py +++ b/modules/sd_models_config.py @@ -1,6 +1,10 @@ import os import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from modules import paths, sd_disable_initialization diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 888f9a30e..dfb478251 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -1,6 +1,10 @@ from collections import namedtuple import numpy as np import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from PIL import Image from modules import devices, processing, images, sd_vae_approx diff --git a/modules/sd_samplers_compvis.py b/modules/sd_samplers_compvis.py index 8de719323..6f08a9022 100644 --- a/modules/sd_samplers_compvis.py +++ b/modules/sd_samplers_compvis.py @@ -4,6 +4,10 @@ import ldm.models.diffusion.plms import numpy as np import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from modules.shared import state from modules import sd_samplers_common, prompt_parser, shared diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py index a30d351fc..5ba34cc33 100644 --- a/modules/sd_samplers_kdiffusion.py +++ b/modules/sd_samplers_kdiffusion.py @@ -1,6 +1,10 @@ from collections import deque import inspect import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import k_diffusion.sampling from modules import prompt_parser, devices, sd_samplers_common diff --git a/modules/sd_vae.py b/modules/sd_vae.py index e5c544487..a13d73be7 100644 --- a/modules/sd_vae.py +++ b/modules/sd_vae.py @@ -3,8 +3,14 @@ import collections import glob from copy import deepcopy from rich import print # pylint: disable=redefined-builtin +from modules import shared import torch -from modules import paths, shared, devices, script_callbacks, sd_models +try: + import intel_extension_for_pytorch as ipex +except: + if shared.cmd_opts.use_ipex: + print("Failed to import IPEX") +from modules import paths, devices, script_callbacks, sd_models vae_ignore_keys = {"model_ema.decay", "model_ema.num_updates"} diff --git a/modules/sd_vae_approx.py b/modules/sd_vae_approx.py index e2f004683..56c3fb15f 100644 --- a/modules/sd_vae_approx.py +++ b/modules/sd_vae_approx.py @@ -1,6 +1,10 @@ import os import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from torch import nn from modules import devices, paths diff --git a/modules/sub_quadratic_attention.py b/modules/sub_quadratic_attention.py index 87c18a38d..0af680de2 100644 --- a/modules/sub_quadratic_attention.py +++ b/modules/sub_quadratic_attention.py @@ -14,6 +14,10 @@ from functools import partial import math from typing import Optional, NamedTuple, List import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from torch import Tensor from torch.utils.checkpoint import checkpoint diff --git a/modules/textual_inversion/dataset.py b/modules/textual_inversion/dataset.py index af9fbcf28..272ae76ea 100644 --- a/modules/textual_inversion/dataset.py +++ b/modules/textual_inversion/dataset.py @@ -2,6 +2,10 @@ import os import numpy as np import PIL import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from PIL import Image from torch.utils.data import Dataset, DataLoader, Sampler from torchvision import transforms diff --git a/modules/textual_inversion/image_embedding.py b/modules/textual_inversion/image_embedding.py index 0ba5db8a4..a2c518af3 100644 --- a/modules/textual_inversion/image_embedding.py +++ b/modules/textual_inversion/image_embedding.py @@ -4,6 +4,10 @@ import numpy as np import zlib from PIL import Image, PngImagePlugin, ImageDraw, ImageFont import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass from modules.shared import opts diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index 36a1e1e17..377b577f0 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -3,6 +3,10 @@ import html import csv from collections import namedtuple import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import tqdm import safetensors.torch from rich import print # pylint: disable=redefined-builtin @@ -434,7 +438,6 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st else: print("No saved optimizer exists in checkpoint") - from modules import shared if shared.cmd_opts.use_ipex: scaler = torch.xpu.amp.GradScaler() else: diff --git a/modules/xlmr.py b/modules/xlmr.py index 9da3161cc..a891beb6d 100644 --- a/modules/xlmr.py +++ b/modules/xlmr.py @@ -1,5 +1,9 @@ from typing import Optional import torch +try: + import intel_extension_for_pytorch as ipex +except: + pass import torch.nn as nn from transformers import XLMRobertaModel,XLMRobertaTokenizer, BertPreTrainedModel, BertModel, BertConfig # pylint: disable=unused-import from transformers.models.xlm_roberta.configuration_xlm_roberta import XLMRobertaConfig diff --git a/webui.py b/webui.py index 7ca052a2e..e9ec96c6d 100644 --- a/webui.py +++ b/webui.py @@ -12,6 +12,10 @@ from modules import timer, errors startup_timer = timer.Timer() import torch # pylint: disable=C0411 +try: + import intel_extension_for_pytorch as ipex +except: + pass import torchvision # pylint: disable=W0611,C0411 import pytorch_lightning # pytorch_lightning should be imported after torch, but it re-enables warnings on import so import once to disable them # pylint: disable=W0611,C0411 logging.getLogger("xformers").addFilter(lambda record: 'A matching Triton is not available' not in record.getMessage()) diff --git a/wiki b/wiki index 6cd8fde16..4cbdffaa9 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 6cd8fde165190057c0849fa6f8dbb183f717b176 +Subproject commit 4cbdffaa95978d0a46758eac4a3fbe689eb4cdcd From 56cdac65929578fd5900ecdad7a8c450e24832bd Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 30 Apr 2023 18:36:52 +0300 Subject: [PATCH 5/6] undo cli --- cli/modules/bench.py | 7 ++-- cli/modules/interrogate-offline.py | 18 ++-------- cli/modules/lora-extract.py | 16 ++------- cli/modules/lora-latents.py | 11 +----- cli/modules/util.py | 20 +---------- cli/random/dynamotest.py | 49 +++++++------------------- cli/train-lora.py | 13 +------ cli/train/latents.py | 11 +----- extensions-builtin/sd-webui-controlnet | 2 +- 9 files changed, 24 insertions(+), 123 deletions(-) diff --git a/cli/modules/bench.py b/cli/modules/bench.py index 18791bfc9..094b73f63 100755 --- a/cli/modules/bench.py +++ b/cli/modules/bench.py @@ -10,7 +10,7 @@ import time from PIL import Image import sdapi from util import Map, log -from modules import shared + options = Map({ 'restore_faces': False, @@ -56,10 +56,7 @@ async def txt2img(): def memstats(): mem = sdapi.getsync('/sdapi/v1/memory') cpu = mem.get('ram', 'unavailable') - if shared.cmd_opts.use_ipex: - gpu = mem.get('xpu', 'unavailable') - else: - gpu = mem.get('cuda', 'unavailable') + gpu = mem.get('cuda', 'unavailable') if 'active' in gpu: gpu['session'] = gpu.pop('active') if 'reserved' in gpu: diff --git a/cli/modules/interrogate-offline.py b/cli/modules/interrogate-offline.py index c2623cda6..6d9ae56fa 100755 --- a/cli/modules/interrogate-offline.py +++ b/cli/modules/interrogate-offline.py @@ -6,12 +6,6 @@ import json import time import argparse import torch -from modules import shared -try: - import intel_extension_for_pytorch as ipex -except: - if shared.cmd_opts.use_ipex: - print("Failed to import IPEX") import filetype from PIL import Image import transformers @@ -25,10 +19,7 @@ model = None processor = None extractor = None dtype = torch.float32 -if shared.cmd_opts.use_ipex: - device = torch.device('xpu') -else: - device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') options = Map({ 'input': '', @@ -138,12 +129,7 @@ def unload_model(): del extractor extractor = None gc.collect() - if shared.cmd_opts.use_ipex: - with torch.no_grad(): - torch.xpu.empty_cache() - with torch.xpu.device('xpu'): - torch.xpu.empty_cache() - elif torch.cuda.is_available(): + if torch.cuda.is_available(): with torch.no_grad(): torch.cuda.empty_cache() with torch.cuda.device('cuda'): diff --git a/cli/modules/lora-extract.py b/cli/modules/lora-extract.py index 9a781789a..102728308 100755 --- a/cli/modules/lora-extract.py +++ b/cli/modules/lora-extract.py @@ -10,12 +10,6 @@ import sys import time import argparse import torch -from modules import shared -try: - import intel_extension_for_pytorch as ipex -except: - if shared.cmd_opts.use_ipex: - print("Failed to import IPEX") import transformers from tqdm import tqdm from util import log @@ -26,10 +20,7 @@ import networks.lora as lora def svd(args): # pylint: disable=redefined-outer-name - if shared.cmd_opts.use_ipex: - device = torch.device('xpu') - else: - device = 'cuda' if torch.cuda.is_available() and args.device == 'cuda' else 'cpu' + device = 'cuda' if torch.cuda.is_available() and args.device == 'cuda' else 'cpu' transformers.logging.set_verbosity_error() CLAMP_QUANTILE = 0.99 MIN_DIFF = 1e-6 @@ -47,10 +38,7 @@ def svd(args): # pylint: disable=redefined-outer-name log.info({ 'loading model': args.tuned }) text_encoder_t, _, unet_t = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.tuned) with torch.no_grad(): - if shared.cmd_opts.use_ipex: - torch.xpu.empty_cache() - else: - torch.cuda.empty_cache() + torch.cuda.empty_cache() # create LoRA network to extract weights: Use dim (rank) as alpha lora_network_o = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_o, unet_o) lora_network_t = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_t, unet_t) diff --git a/cli/modules/lora-latents.py b/cli/modules/lora-latents.py index 7e701df12..d556d596b 100755 --- a/cli/modules/lora-latents.py +++ b/cli/modules/lora-latents.py @@ -10,12 +10,6 @@ import warnings import cv2 import numpy as np import torch -from modules import shared -try: - import intel_extension_for_pytorch as ipex -except: - if shared.cmd_opts.use_ipex: - print("Failed to import IPEX") from PIL import Image from torchvision import transforms from tqdm import tqdm @@ -26,10 +20,7 @@ import library.model_util as model_util import library.train_util as train_util warnings.filterwarnings('ignore') -if shared.cmd_opts.use_ipex: - device = torch.device('xpu') -else: - device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') options = Map({ 'batch': 1, 'input': '', diff --git a/cli/modules/util.py b/cli/modules/util.py index d48887961..5ab9dee6b 100755 --- a/cli/modules/util.py +++ b/cli/modules/util.py @@ -45,25 +45,7 @@ def get_memory(): try: import torch from modules import shared - if shared.cmd_opts.use_ipex: - import intel_extension_for_pytorch as ipex - s = torch.xpu.mem_get_info() - gpu = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) } - s = dict(torch.xpu.memory_stats('xpu')) - allocated = { 'current': gb(s['allocated_bytes.all.current']), 'peak': gb(s['allocated_bytes.all.peak']) } - reserved = { 'current': gb(s['reserved_bytes.all.current']), 'peak': gb(s['reserved_bytes.all.peak']) } - active = { 'current': gb(s['active_bytes.all.current']), 'peak': gb(s['active_bytes.all.peak']) } - inactive = { 'current': gb(s['inactive_split_bytes.all.current']), 'peak': gb(s['inactive_split_bytes.all.peak']) } - warnings = { 'retries': s['num_alloc_retries'], 'oom': s['num_ooms'] } - mem.update({ - 'gpu': gpu, - 'gpu-active': active, - 'gpu-allocated': allocated, - 'gpu-reserved': reserved, - 'gpu-inactive': inactive, - 'events': warnings, - }) - elif torch.cuda.is_available(): + if torch.cuda.is_available(): s = torch.cuda.mem_get_info() gpu = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) } s = dict(torch.cuda.memory_stats('cuda')) diff --git a/cli/random/dynamotest.py b/cli/random/dynamotest.py index 556ae96a8..82b1143c6 100755 --- a/cli/random/dynamotest.py +++ b/cli/random/dynamotest.py @@ -7,14 +7,9 @@ import warnings import numpy as np import torch -from modules import shared -try: - import intel_extension_for_pytorch as ipex -except: - if shared.cmd_opts.use_ipex: - print("Failed to import IPEX") from torchvision.models import resnet18 + print('torch:', torch.__version__) try: import torch._dynamo as dynamo # must be imported explicitly or namespace is not found @@ -29,42 +24,24 @@ warnings.filterwarnings('ignore', category=UserWarning) # disable those for now def timed(fn): # returns the result of running `fn()` and the time it took for `fn()` to run in ms using CUDA events - if shared.cmd_opts.use_ipex: - start = torch.xpu.Event(enable_timing=True) - end = torch.xpu.Event(enable_timing=True) - start.record() - result = fn() - end.record() - torch.xpu.synchronize() - return result, start.elapsed_time(end) - else: - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - start.record() - result = fn() - end.record() - torch.cuda.synchronize() - return result, start.elapsed_time(end) + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + result = fn() + end.record() + torch.cuda.synchronize() + return result, start.elapsed_time(end) def generate_data(b): - if shared.cmd_opts.use_ipex: - return ( - torch.randn(b, 3, 128, 128).to(torch.float32).xpu(), - torch.randint(1000, (b,)).xpu(), - ) - else: - return ( - torch.randn(b, 3, 128, 128).to(torch.float32).cuda(), - torch.randint(1000, (b,)).cuda(), - ) + return ( + torch.randn(b, 3, 128, 128).to(torch.float32).cuda(), + torch.randint(1000, (b,)).cuda(), + ) def init_model(): - if shared.cmd_opts.use_ipex: - return resnet18().to(torch.float32).xpu() - else: - return resnet18().to(torch.float32).cuda() + return resnet18().to(torch.float32).cuda() def eval(mod, inp): diff --git a/cli/train-lora.py b/cli/train-lora.py index f1e295a35..6f82063a6 100755 --- a/cli/train-lora.py +++ b/cli/train-lora.py @@ -23,12 +23,6 @@ import shutil import argparse import tempfile import torch -from modules import shared -try: - import intel_extension_for_pytorch as ipex -except: - if shared.cmd_opts.use_ipex: - print("Failed to import IPEX") import logging import importlib import transformers @@ -123,12 +117,7 @@ options = Map({ def mem_stats(): gc.collect() - if shared.cmd_opts.use_ipex: - with torch.no_grad(): - torch.xpu.empty_cache() - with torch.xpu.device('xpu'): - torch.cuda.empty_cache() - elif torch.cuda.is_available(): + if torch.cuda.is_available(): with torch.no_grad(): torch.cuda.empty_cache() with torch.cuda.device('cuda'): diff --git a/cli/train/latents.py b/cli/train/latents.py index 715f92aba..94249b18a 100755 --- a/cli/train/latents.py +++ b/cli/train/latents.py @@ -10,12 +10,6 @@ import warnings import cv2 import numpy as np import torch -from modules import shared -try: - import intel_extension_for_pytorch as ipex -except: - if shared.cmd_opts.use_ipex: - print("Failed to import IPEX") from PIL import Image from torchvision import transforms from tqdm import tqdm @@ -34,10 +28,7 @@ import library.model_util as model_util import library.train_util as train_util warnings.filterwarnings('ignore') -if shared.cmd_opts.use_ipex: - device = torch.device('xpu') -else: - device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') options = Map({ 'batch': 1, 'input': '', diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index d2da774a4..af4720780 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit d2da774a40ff9c3770e21f71fb516403022fc3f6 +Subproject commit af4720780f10d912789cbd6db1fbc6d2f0afc533 From 185b796991e6ae8f71c03fea826581f0799ce9b4 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 30 Apr 2023 18:37:58 +0300 Subject: [PATCH 6/6] undo cli --- cli/modules/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/modules/util.py b/cli/modules/util.py index 5ab9dee6b..479b77233 100755 --- a/cli/modules/util.py +++ b/cli/modules/util.py @@ -44,7 +44,6 @@ def get_memory(): mem.update({ 'ram': e }) try: import torch - from modules import shared if torch.cuda.is_available(): s = torch.cuda.mem_get_info() gpu = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }