More patches and Import IPEX after Torch

modules/api/api.py

@@ -573,7 +573,24 @@ class Api:
             ram = { 'error': f'{err}' }
         try:
             import torch
-            if torch.cuda.is_available():
+            if shared.cmd_opts.use_ipex():
+                import intel_extension_for_pytorch as ipex
+                system = { 'free': (torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), 'used': torch.xpu.memory_allocated(), 'total': torch.xpu.get_device_properties("xpu").total_memory }
+                s = dict(torch.xpu.memory_stats("xpu"))
+                allocated = { 'current': s['allocated_bytes.all.current'], 'peak': s['allocated_bytes.all.peak'] }
+                reserved = { 'current': s['reserved_bytes.all.current'], 'peak': s['reserved_bytes.all.peak'] }
+                active = { 'current': s['active_bytes.all.current'], 'peak': s['active_bytes.all.peak'] }
+                inactive = { 'current': s['inactive_split_bytes.all.current'], 'peak': s['inactive_split_bytes.all.peak'] }
+                warnings = { 'retries': s['num_alloc_retries'], 'oom': s['num_ooms'] }
+                cuda = {
+                    'system': system,
+                    'active': active,
+                    'allocated': allocated,
+                    'reserved': reserved,
+                    'inactive': inactive,
+                    'events': warnings,
+                }
+            elif torch.cuda.is_available():
                 s = torch.cuda.mem_get_info()
                 system = { 'free': s[0], 'used': s[1] - s[0], 'total': s[1] }
                 s = dict(torch.cuda.memory_stats(shared.device))

modules/codeformer/codeformer_arch.py

@@ -3,6 +3,10 @@
 import math
 import numpy as np
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from torch import nn, Tensor
 import torch.nn.functional as F
 from typing import Optional, List

modules/codeformer/vqgan_arch.py

@@ -7,6 +7,10 @@ https://github.com/samb-t/unleashing-transformers/blob/master/models/vqgan.py
 '''
 import numpy as np
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.nn as nn
 import torch.nn.functional as F
 import copy

modules/codeformer_model.py

@@ -3,6 +3,10 @@ import sys
 
 import cv2
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 import modules.face_restoration
 from modules import shared, devices, modelloader, errors
@@ -103,8 +107,7 @@ def setup_model(dirname):
                             output = self.net(cropped_face_t, w=w if w is not None else shared.opts.code_former_weight, adain=True)[0]
                             restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1))
                         del output
-                        from modules import shared
-                        if shared.cmd_opts.use_ipex:
+                        if cmd_opts.use_ipex:
                             torch.xpu.empty_cache()
                         else:
                             torch.cuda.empty_cache()

modules/deepbooru.py

@@ -2,6 +2,10 @@ import os
 import re
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import numpy as np
 
 from modules import modelloader, paths, deepbooru_model, devices, images, shared

modules/deepbooru_model.py

@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.nn as nn
 import torch.nn.functional as F
 

modules/devices.py

@@ -5,8 +5,7 @@ from modules import shared
 try:
     import intel_extension_for_pytorch as ipex
 except:
-    if shared.cmd_opts.use_ipex:
-        print("Failed to import IPEX")
+    pass
 
 if sys.platform == "darwin":
     from modules import mac_specific

modules/esrgan_model.py

@@ -2,6 +2,10 @@ import os
 
 import numpy as np
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from PIL import Image
 from basicsr.utils.download_util import load_file_from_url
 

modules/esrgan_model_arch.py

@@ -2,6 +2,10 @@
 
 import math
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.nn as nn
 import torch.nn.functional as F
 

modules/extras.py

@@ -4,6 +4,10 @@ import html
 import shutil
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import tqdm
 import gradio as gr
 import safetensors.torch

modules/hypernetworks/hypernetwork.py

@@ -8,6 +8,10 @@ import inspect
 
 import modules.textual_inversion.dataset
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import tqdm
 from einops import rearrange, repeat
 from ldm.util import default
@@ -591,7 +595,10 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
             print("Cannot resume from saved optimizer!")
             print(e)
 
-    scaler = torch.cuda.amp.GradScaler()
+    if shared.cmd_opts.use_ipex:
+        scaler = torch.xpu.amp.GradScaler()
+    else:
+        scaler = torch.cuda.amp.GradScaler()
     
     batch_size = ds.batch_size
     gradient_step = ds.gradient_step
@@ -708,7 +715,9 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
                     hypernetwork.eval()
                     rng_state = torch.get_rng_state()
                     cuda_rng_state = None
-                    if torch.cuda.is_available():
+                    if shared.cmd_opts.use_ipex:
+                        cuda_rng_state = torch.xpu.get_rng_state_all()
+                    elif torch.cuda.is_available():
                         cuda_rng_state = torch.cuda.get_rng_state_all()
                     shared.sd_model.cond_stage_model.to(devices.device)
                     shared.sd_model.first_stage_model.to(devices.device)
@@ -745,7 +754,9 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
                         shared.sd_model.cond_stage_model.to(devices.cpu)
                         shared.sd_model.first_stage_model.to(devices.cpu)
                     torch.set_rng_state(rng_state)
-                    if torch.cuda.is_available():
+                    if shared.cmd_opts.use_ipex:
+                        torch.xpu.set_rng_state_all(cuda_rng_state)
+                    elif torch.cuda.is_available():
                         torch.cuda.set_rng_state_all(cuda_rng_state)
                     hypernetwork.train()
                     if image is not None:

modules/interrogate.py

@@ -5,6 +5,10 @@ from pathlib import Path
 import re
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.hub
 
 from torchvision import transforms

modules/lowvram.py

@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from modules import devices
 
 module_in_gpu = None

modules/mac_specific.py

@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import platform
 from modules.sd_hijack_utils import CondFunc
 from packaging import version

modules/memmon.py

@@ -2,6 +2,12 @@ import threading
 import time
 from collections import defaultdict
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
+
+from modules import shared
 
 
 class MemUsageMonitor(threading.Thread):
@@ -19,7 +25,6 @@ class MemUsageMonitor(threading.Thread):
         self.daemon = True
         self.run_flag = threading.Event()
         self.data = defaultdict(int)
-        from modules import shared
         if not torch.cuda.is_available() or not shared.cmd_opts.use_ipex:
             self.disabled = True
         else:
@@ -40,9 +45,8 @@ class MemUsageMonitor(threading.Thread):
                     self.disabled = True
 
     def cuda_mem_get_info(self):
-        from modules import shared
         if shared.cmd_opts.use_ipex:
-            return torch.xpu.mem_get_info("xpu")
+            return [(torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), torch.xpu.get_device_properties("xpu").total_memory]
         else:
             index = self.device.index if self.device.index is not None else torch.cuda.current_device()
             return torch.cuda.mem_get_info(index)
@@ -52,7 +56,6 @@ class MemUsageMonitor(threading.Thread):
             return
         while True:
             self.run_flag.wait()
-            from modules import shared
             if shared.cmd_opts.use_ipex:
                 torch.xpu.reset_peak_memory_stats()
             else:
@@ -72,7 +75,6 @@ class MemUsageMonitor(threading.Thread):
         for k, v in self.read().items():
             print(k, -(v // -(1024 ** 2)))
         print(self, 'raw torch memory stats:')
-        from modules import shared
         if shared.cmd_opts.use_ipex:
             tm = torch.xpu.memory_stats("xpu")
         else:
@@ -95,7 +97,6 @@ class MemUsageMonitor(threading.Thread):
             self.data["free"] = free
             self.data["total"] = total
 
-            from modules import shared
             if shared.cmd_opts.use_ipex:
                 torch_stats = torch.xpu.memory_stats("xpu")
             else:

modules/models/diffusion/ddpm_edit.py

@@ -10,6 +10,10 @@ https://github.com/CompVis/taming-transformers
 # See more details in LICENSE.
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.nn as nn
 import numpy as np
 import pytorch_lightning as pl

modules/models/diffusion/uni_pc/sampler.py

@@ -2,6 +2,10 @@
 
 import numpy as np
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 from .uni_pc import NoiseScheduleVP, model_wrapper, UniPC
 from modules import shared, devices

modules/models/diffusion/uni_pc/uni_pc.py

@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.nn.functional as F
 import math
 import time

modules/processing.py

@@ -8,6 +8,10 @@ from typing import Any, Dict, List
 
 import psutil
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import numpy as np
 from PIL import Image, ImageFilter, ImageOps
 import cv2
@@ -55,10 +59,8 @@ def memory_stats():
     except Exception as e:
         mem.update({ 'ram': e })
     try:
-        from modules import shared
-        if shared.cmd_opts.use_ipex:
-            s = torch.xpu.mem_get_info()
-            gpu = { 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }
+        if cmd_opts.use_ipex:
+            gpu = { 'used': gb(torch.xpu.memory_allocated()), 'total': gb(torch.xpu.get_device_properties("xpu").total_memory) }
             s = dict(torch.xpu.memory_stats("xpu"))
             mem.update({
                 'gpu': gpu,

modules/prompt_parser.py

@@ -368,3 +368,7 @@ if __name__ == "__main__":
     doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
 else:
     import torch  # doctest faster
+    try:
+        import intel_extension_for_pytorch as ipex
+    except:
+        pass

modules/safe.py

@@ -6,6 +6,10 @@ import zipfile
 import re
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import numpy
 import _codecs
 

modules/sd_disable_initialization.py

@@ -1,6 +1,10 @@
 import ldm.modules.encoders.modules
 import open_clip
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import transformers.utils.hub
 
 

modules/sd_hijack.py

@@ -1,6 +1,10 @@
 from types import MethodType
 from rich import print # pylint: disable=redefined-builtin
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from torch.nn.functional import silu
 import ldm.modules.attention
 import ldm.modules.diffusionmodules.model

modules/sd_hijack_clip.py

@@ -2,6 +2,10 @@ import math
 from collections import namedtuple
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 from modules import prompt_parser, devices, sd_hijack
 from modules.shared import opts

modules/sd_hijack_inpainting.py

@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 import ldm.models.diffusion.ddpm
 import ldm.models.diffusion.ddim

modules/sd_hijack_open_clip.py

@@ -1,5 +1,9 @@
 import open_clip.tokenizer
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 from modules import sd_hijack_clip, devices
 

modules/sd_hijack_optimizations.py

@@ -2,6 +2,10 @@ import math
 import psutil
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from torch import einsum
 
 from ldm.util import default
@@ -26,7 +30,7 @@ def get_available_vram():
         stats = torch.xpu.memory_stats("xpu")
         mem_active = stats['active_bytes.all.current']
         mem_reserved = stats['reserved_bytes.all.current']
-        mem_free_xpu, _ = torch.xpu.mem_get_info("xpu")
+        mem_free_xpu, _ = [(torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), torch.xpu.get_device_properties("xpu").total_memory]
         mem_free_torch = mem_reserved - mem_active
         mem_free_total = mem_free_xpu + mem_free_torch
         return mem_free_total
@@ -201,7 +205,7 @@ def einsum_op_cuda(q, k, v):
         stats = torch.xpu.memory_stats("xpu")
         mem_active = stats['active_bytes.all.current']
         mem_reserved = stats['reserved_bytes.all.current']
-        mem_free_xpu, _ = torch.xpu.mem_get_info("xpu")
+        mem_free_xpu, _ = [(torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), torch.xpu.get_device_properties("xpu").total_memory]
         mem_free_torch = mem_reserved - mem_active
         mem_free_total = mem_free_xpu + mem_free_torch
         # Divide factor of safety as there's copying and fragmentation

modules/sd_hijack_unet.py

@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from packaging import version
 
 from modules import devices

modules/sd_hijack_xlmr.py

@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 from modules import sd_hijack_clip, devices
 

modules/sd_models.py

@@ -8,6 +8,10 @@ from os import mkdir
 from urllib import request
 from rich import print, progress # pylint: disable=redefined-builtin
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import safetensors.torch
 from omegaconf import OmegaConf
 import tomesd

modules/sd_models_config.py

@@ -1,6 +1,10 @@
 import os
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 from modules import paths, sd_disable_initialization
 

modules/sd_samplers_common.py

@@ -1,6 +1,10 @@
 from collections import namedtuple
 import numpy as np
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from PIL import Image
 from modules import devices, processing, images, sd_vae_approx
 

modules/sd_samplers_compvis.py

@@ -4,6 +4,10 @@ import ldm.models.diffusion.plms
 
 import numpy as np
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 from modules.shared import state
 from modules import sd_samplers_common, prompt_parser, shared

modules/sd_samplers_kdiffusion.py

@@ -1,6 +1,10 @@
 from collections import deque
 import inspect
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import k_diffusion.sampling
 from modules import prompt_parser, devices, sd_samplers_common
 

modules/sd_vae.py

@@ -3,8 +3,14 @@ import collections
 import glob
 from copy import deepcopy
 from rich import print # pylint: disable=redefined-builtin
+from modules import shared
 import torch
-from modules import paths, shared, devices, script_callbacks, sd_models
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    if shared.cmd_opts.use_ipex:
+        print("Failed to import IPEX")
+from modules import paths, devices, script_callbacks, sd_models
 
 
 vae_ignore_keys = {"model_ema.decay", "model_ema.num_updates"}

modules/sd_vae_approx.py

@@ -1,6 +1,10 @@
 import os
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from torch import nn
 from modules import devices, paths
 

modules/sub_quadratic_attention.py

@@ -14,6 +14,10 @@ from functools import partial
 import math
 from typing import Optional, NamedTuple, List
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from torch import Tensor
 from torch.utils.checkpoint import checkpoint
 

modules/textual_inversion/dataset.py

@@ -2,6 +2,10 @@ import os
 import numpy as np
 import PIL
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from PIL import Image
 from torch.utils.data import Dataset, DataLoader, Sampler
 from torchvision import transforms

modules/textual_inversion/image_embedding.py

@@ -4,6 +4,10 @@ import numpy as np
 import zlib
 from PIL import Image, PngImagePlugin, ImageDraw, ImageFont
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from modules.shared import opts
 
 

modules/textual_inversion/textual_inversion.py

@@ -3,6 +3,10 @@ import html
 import csv
 from collections import namedtuple
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import tqdm
 import safetensors.torch
 from rich import print # pylint: disable=redefined-builtin
@@ -434,7 +438,6 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
         else:
             print("No saved optimizer exists in checkpoint")
 
-    from modules import shared
     if shared.cmd_opts.use_ipex:
         scaler = torch.xpu.amp.GradScaler()
     else:

modules/xlmr.py

@@ -1,5 +1,9 @@
 from typing import Optional
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.nn as nn
 from transformers import XLMRobertaModel,XLMRobertaTokenizer, BertPreTrainedModel, BertModel, BertConfig # pylint: disable=unused-import
 from transformers.models.xlm_roberta.configuration_xlm_roberta import XLMRobertaConfig

webui.py

@@ -12,6 +12,10 @@ from modules import timer, errors
 startup_timer = timer.Timer()
 
 import torch # pylint: disable=C0411
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torchvision # pylint: disable=W0611,C0411
 import pytorch_lightning # pytorch_lightning should be imported after torch, but it re-enables warnings on import so import once to disable them # pylint: disable=W0611,C0411
 logging.getLogger("xformers").addFilter(lambda record: 'A matching Triton is not available' not in record.getMessage())