From a49a8f8b462b96af9eb682a6bbcb6cd96f276868 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Tue, 25 Apr 2023 01:43:19 +0900
Subject: [PATCH 01/69] First DirectML implementation. Unstable and not tested.

---
 TODO_DML.md                        | 22 +++++++++
 modules/devices.py                 | 16 ++++++-
 modules/realesrgan_model.py        | 71 +++++++++++++++++++++++++++++-
 modules/sd_hijack.py               |  3 ++
 modules/sd_hijack_inpainting.py    |  1 +
 modules/sd_hijack_optimizations.py | 10 +++++
 modules/shared.py                  |  2 +-
 setup.py                           | 38 +++++++++++-----
 8 files changed, 149 insertions(+), 14 deletions(-)
 create mode 100644 TODO_DML.md

diff --git a/TODO_DML.md b/TODO_DML.md
new file mode 100644
index 000000000..af6559ac1
--- /dev/null
+++ b/TODO_DML.md
@@ -0,0 +1,22 @@
+# TODO
+
+## Issues
+
+Stuff to be fixed...
+
+- `mat1 and mat2 must have the same dtype` error (half mode)
+- Some samplers won't work (test later)
+
+## Something needs discussion
+
+- About memory optimization.
+
+Basically, we cannot get detailed vram information from `torch-directml`.
+
+It has `gpu_memory` method which returns an array contains used memory size, but it is almostly useless without any other information.
+
+What should we do?
+
+1. Use any fixed value as the available memory capacity.
+2. Use `atiadlxx`(AMD/ATI GPU driver library) to infer vram information as similar as possible to the actual value. (works for AMDGPUs)
+3. or another better way.
diff --git a/modules/devices.py b/modules/devices.py
index 7c869d60a..5f5e18768 100644
--- a/modules/devices.py
+++ b/modules/devices.py
@@ -27,12 +27,26 @@ def get_cuda_device_string():
     return "cuda"
 
 
+def get_dml_device_string():
+    from modules import shared
+    if shared.cmd_opts.device_id is not None:
+        return f"privateuseone:{shared.cmd_opts.device_id}"
+    return "privateuseone"
+
+
 def get_optimal_device_name():
     if torch.cuda.is_available():
         return get_cuda_device_string()
     if has_mps():
         return "mps"
-    return "cpu"
+    try:
+        import torch_directml
+        if torch_directml.is_available():
+            return get_dml_device_string()
+        else:
+            return "cpu"
+    except:
+        return "cpu"
 
 
 def get_optimal_device():
diff --git a/modules/realesrgan_model.py b/modules/realesrgan_model.py
index 86007f906..ac1102e38 100644
--- a/modules/realesrgan_model.py
+++ b/modules/realesrgan_model.py
@@ -6,10 +6,76 @@ from PIL import Image
 from basicsr.utils.download_util import load_file_from_url
 
 from modules.upscaler import Upscaler, UpscalerData
-from modules.shared import cmd_opts, opts
+from modules.shared import cmd_opts, opts, device
 import modules.errors as errors
 
 
+# DML ISSUE: Some tensors turn 0 after Extended Slices.
+def realesrgan_tile_process_dml_fix(self):
+    import math
+    import torch
+    batch, channel, height, width = self.img.shape
+    output_height = height * self.scale
+    output_width = width * self.scale
+    output_shape = (batch, channel, output_height, output_width)
+
+    # start with black image
+    self.output = self.img.new_zeros(output_shape, device='cpu' if self.device.type == 'privateuseone' else self.device)
+    tiles_x = math.ceil(width / self.tile_size)
+    tiles_y = math.ceil(height / self.tile_size)
+
+    # loop over all tiles
+    for y in range(tiles_y):
+        for x in range(tiles_x):
+            # extract tile from input image
+            ofs_x = x * self.tile_size
+            ofs_y = y * self.tile_size
+            # input tile area on total image
+            input_start_x = ofs_x
+            input_end_x = min(ofs_x + self.tile_size, width)
+            input_start_y = ofs_y
+            input_end_y = min(ofs_y + self.tile_size, height)
+
+            # input tile area on total image with padding
+            input_start_x_pad = max(input_start_x - self.tile_pad, 0)
+            input_end_x_pad = min(input_end_x + self.tile_pad, width)
+            input_start_y_pad = max(input_start_y - self.tile_pad, 0)
+            input_end_y_pad = min(input_end_y + self.tile_pad, height)
+
+            # input tile dimensions
+            input_tile_width = input_end_x - input_start_x
+            input_tile_height = input_end_y - input_start_y
+            tile_idx = y * tiles_x + x + 1
+            input_tile = self.img[:, :, input_start_y_pad:input_end_y_pad, input_start_x_pad:input_end_x_pad]
+
+            # upscale tile
+            try:
+                with torch.no_grad():
+                    output_tile = self.model(input_tile)
+                    output_tile = output_tile.cpu()
+            except RuntimeError as error:
+                print('Error', error)
+            print(f'\tTile {tile_idx}/{tiles_x * tiles_y}')
+
+            # output tile area on total image
+            output_start_x = input_start_x * self.scale
+            output_end_x = input_end_x * self.scale
+            output_start_y = input_start_y * self.scale
+            output_end_y = input_end_y * self.scale
+
+            # output tile area without padding
+            output_start_x_tile = (input_start_x - input_start_x_pad) * self.scale
+            output_end_x_tile = output_start_x_tile + input_tile_width * self.scale
+            output_start_y_tile = (input_start_y - input_start_y_pad) * self.scale
+            output_end_y_tile = output_start_y_tile + input_tile_height * self.scale
+
+            # put tile into output image
+            self.output[:, :, output_start_y:output_end_y,
+                        output_start_x:output_end_x] = output_tile[:, :, output_start_y_tile:output_end_y_tile,
+                                                                    output_start_x_tile:output_end_x_tile]
+    self.output = self.output.to(self.device)
+
+
 class UpscalerRealESRGAN(Upscaler):
     def __init__(self, path):
         self.name = "RealESRGAN"
@@ -37,6 +103,8 @@ class UpscalerRealESRGAN(Upscaler):
 
         try:
             from realesrgan import RealESRGANer
+            if device.type == 'privateuseone':
+                RealESRGANer.tile_process = realesrgan_tile_process_dml_fix
         except:
             print("Error importing Real-ESRGAN:", file=sys.stderr)
             return img
@@ -53,6 +121,7 @@ class UpscalerRealESRGAN(Upscaler):
             half=not cmd_opts.no_half and not opts.upcast_sampling,
             tile=opts.ESRGAN_tile,
             tile_pad=opts.ESRGAN_tile_overlap,
+            device=device,
         )
 
         upsampled = upsampler.enhance(np.array(img), outscale=info.scale)[0]
diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index f817b7afd..cd859a6c3 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -257,6 +257,9 @@ class EmbeddingsWithFixes(torch.nn.Module):
             for offset, embedding in fixes:
                 emb = devices.cond_cast_unet(embedding.vec)
                 emb_len = min(tensor.shape[0] - offset - 1, emb.shape[0])
+                # DML ISSUE: type mismatch on half mode
+                if tensor.dtype == torch.float16 and emb.dtype == torch.float32 and not shared.cmd_opts.no_half:
+                    emb = emb.half()
                 tensor = torch.cat([tensor[0:offset + 1], emb[0:emb_len], tensor[offset + 1 + emb_len:]])
 
             vecs.append(tensor)
diff --git a/modules/sd_hijack_inpainting.py b/modules/sd_hijack_inpainting.py
index 4b23c132d..0c37fef89 100644
--- a/modules/sd_hijack_inpainting.py
+++ b/modules/sd_hijack_inpainting.py
@@ -53,6 +53,7 @@ def p_sample_plms(self, x, c, t, index, repeat_noise=False, use_original_steps=F
 
     def get_x_prev_and_pred_x0(e_t, index):
         # select parameters corresponding to the currently considered timestep
+        print(alphas[index]) # DML ISSUE: PLMS Sampling does not work without this print.
         a_t = torch.full((b, 1, 1, 1), alphas[index], device=device)
         a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device)
         sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device)
diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py
index 776d91d6c..0f1f2a757 100644
--- a/modules/sd_hijack_optimizations.py
+++ b/modules/sd_hijack_optimizations.py
@@ -30,6 +30,9 @@ def get_available_vram():
         mem_free_torch = mem_reserved - mem_active
         mem_free_total = mem_free_cuda + mem_free_torch
         return mem_free_total
+    elif shared.device.type == 'privateuseone':
+        # DML ISSUE: There's no way to get any memory info.
+        return 1048576
     else:
         return psutil.virtual_memory().available
 
@@ -195,6 +198,10 @@ def einsum_op_cuda(q, k, v):
     # Divide factor of safety as there's copying and fragmentation
     return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20))
 
+def einsum_op_dml(q, k, v):
+    # DML ISSUE: There's no way to get any memory info.
+    return einsum_op_tensor_mem(q, k, v, 1024)
+
 def einsum_op(q, k, v):
     if q.device.type == 'cuda':
         return einsum_op_cuda(q, k, v)
@@ -204,6 +211,9 @@ def einsum_op(q, k, v):
             return einsum_op_mps_v1(q, k, v)
         return einsum_op_mps_v2(q, k, v)
 
+    if q.device.type == 'privateuseone':
+        return einsum_op_dml(q, k, v)
+
     # Smaller slices are faster due to L2/L3/SLC caches.
     # Tested on i7 with 8MB L3 cache.
     return einsum_op_tensor_mem(q, k, v, 32)
diff --git a/modules/shared.py b/modules/shared.py
index a2e202735..f147317cd 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -422,7 +422,7 @@ options_templates.update(options_section(('ui', "Live previews"), {
     "live_previews_enable": OptionInfo(True, "Show live previews of the created image"),
     "show_progress_grid": OptionInfo(True, "Show previews of all images generated in a batch as a grid"),
     "show_progress_every_n_steps": OptionInfo(1, "Show new live preview image every N sampling steps. Set to -1 to show after completion of batch.", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}),
-    "show_progress_type": OptionInfo("Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}),
+    "show_progress_type": OptionInfo("Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}), # DML ISSUE: Approx NN does not work well on DirectML device.
     "live_preview_content": OptionInfo("Combined", "Live preview subject", gr.Radio, {"choices": ["Combined", "Prompt", "Negative prompt"]}),
     "live_preview_refresh_period": OptionInfo(250, "Progressbar/preview update period, in milliseconds")
 }))
diff --git a/setup.py b/setup.py
index 73269b0df..aa01a3de5 100644
--- a/setup.py
+++ b/setup.py
@@ -4,6 +4,7 @@ import json
 import time
 import shutil
 import logging
+import platform
 import subprocess
 
 try:
@@ -20,7 +21,7 @@ class Dot(dict): # dot notation access to dictionary attributes
 
 
 log = logging.getLogger("sd")
-args = Dot({ 'debug': False, 'upgrade': False, 'noupdate': False, 'skip-extensions': False, 'skip-requirements': False, 'reset': False })
+args = Dot({ 'debug': False, 'upgrade': False, 'noupdate': False, 'nodirectml': False, 'skip-extensions': False, 'skip-requirements': False, 'reset': False })
 quick_allowed = True
 errors = 0
 opts = {}
@@ -188,17 +189,21 @@ def check_torch():
         torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.4.2')
         xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
     else:
-        log.info('Using CPU-only Torch')
-        torch_command = os.environ.get('TORCH_COMMAND', 'torch torchaudio torchvision')
-        xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
+        machine = platform.machine()
+        if 'arm' not in machine and 'aarch' not in machine and not args.nodirectml: # torch-directml is available on AMD64
+            log.info('Using DirectML Backend')
+            torch_command = os.environ.get('TORCH_COMMAND', 'torch==1.13.1 torchvision==0.14.1 torch-directml')
+            xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
+        else:
+            log.info('Using CPU-only Torch')
+            torch_command = os.environ.get('TORCH_COMMAND', 'torch torchaudio torchvision')
+            xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
     if 'torch' in torch_command:
         install(torch_command, 'torch torchvision torchaudio')
     try:
         import torch
         log.info(f'Torch {torch.__version__}')
-        if not torch.cuda.is_available():
-            log.warning("Torch repoorts CUDA not available")
-        else:
+        if torch.cuda.is_available():
             if torch.version.cuda:
                 log.info(f'Torch backend: nVidia CUDA {torch.version.cuda} cuDNN {torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else "N/A"}')
             elif torch.version.hip:
@@ -207,6 +212,16 @@ def check_torch():
                 log.warning('Unknown Torch backend')
             for device in [torch.cuda.device(i) for i in range(torch.cuda.device_count())]:
                 log.info(f'Torch detected GPU: {torch.cuda.get_device_name(device)} VRAM {round(torch.cuda.get_device_properties(device).total_memory / 1024 / 1024)} Arch {torch.cuda.get_device_capability(device)} Cores {torch.cuda.get_device_properties(device).multi_processor_count}')
+        else:
+            try:
+                import torch_directml
+                import pkg_resources
+                version = pkg_resources.get_distribution("torch-directml")
+                log.info(f'Torch backend: DirectML ({version})')
+                for i in range(0, torch_directml.device_count()):
+                    log.info(f'Torch detected GPU: {torch_directml.device_name(i)}')
+            except:
+                log.warning("Torch repoorts CUDA not available")
     except Exception as e:
         log.error(f'Could not load torch: {e}')
         exit(1)
@@ -239,14 +254,14 @@ def install_repositories():
         return os.path.join(os.path.dirname(__file__), 'repositories', name)
     log.info('Installing repositories')
     os.makedirs(os.path.join(os.path.dirname(__file__), 'repositories'), exist_ok=True)
-    stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git")
-    stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf")
+    stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") # DML TODO: check samplers work well
+    stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "d4c168b2ad29d82e5fdfea4d598075f40a3b0341")
     clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit)
     taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git")
     taming_transformers_commit = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "3ba01b241669f5ade541ce990f7650a3b8f65318")
     clone(taming_transformers_repo, d('taming-transformers'), taming_transformers_commit)
-    k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git')
-    k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919")
+    k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git') # DML TODO: check samplers work well
+    k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "47b6ef08bca986ff5e72815e74a419ef6616bdbb")
     clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit)
     codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git')
     codeformer_commit = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af")
@@ -481,6 +496,7 @@ def parse_args():
     parser.add_argument('--reset', default = False, action='store_true', help = "Reset main repository to latest version, default: %(default)s")
     parser.add_argument('--upgrade', default = False, action='store_true', help = "Upgrade main repository to latest version, default: %(default)s")
     parser.add_argument('--noupdate', default = False, action='store_true', help = "Skip update of extensions and submodules, default: %(default)s")
+    parser.add_argument('--nodirectml', default = False, action='store_true', help = "Although nVidia and AMD toolkit aren't detected, use CPU not DirectML, default: %(default)s")
     parser.add_argument('--skip-requirements', default = False, action='store_true', help = "Skips checking and installing requirements, default: %(default)s")
     parser.add_argument('--skip-extensions', default = False, action='store_true', help = "Skips running individual extension installers, default: %(default)s")
     parser.add_argument('--skip-git', default = False, action='store_true', help = "Skips running all GIT operations, default: %(default)s")

From 836324cd2c5ff362d5d15b09c395ae39134d07d4 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Tue, 25 Apr 2023 01:55:27 +0900
Subject: [PATCH 02/69] Fix PLMS & DPM & DDIM. Unstable and not tested.

---
 modules/realesrgan_model.py     |  2 +-
 modules/sd_hijack.py            |  2 +-
 modules/sd_hijack_inpainting.py |  2 +-
 setup.py                        | 21 +++++++++++++++------
 4 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/modules/realesrgan_model.py b/modules/realesrgan_model.py
index ac1102e38..1d48ac42c 100644
--- a/modules/realesrgan_model.py
+++ b/modules/realesrgan_model.py
@@ -10,7 +10,7 @@ from modules.shared import cmd_opts, opts, device
 import modules.errors as errors
 
 
-# DML ISSUE: Some tensors turn 0 after Extended Slices.
+# DML Solution: Some tensors turn 0 after Extended Slices. Move output to cpu and get it back.
 def realesrgan_tile_process_dml_fix(self):
     import math
     import torch
diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index cd859a6c3..4a68216df 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -257,7 +257,7 @@ class EmbeddingsWithFixes(torch.nn.Module):
             for offset, embedding in fixes:
                 emb = devices.cond_cast_unet(embedding.vec)
                 emb_len = min(tensor.shape[0] - offset - 1, emb.shape[0])
-                # DML ISSUE: type mismatch on half mode
+                # DML Solution: type mismatch on half mode
                 if tensor.dtype == torch.float16 and emb.dtype == torch.float32 and not shared.cmd_opts.no_half:
                     emb = emb.half()
                 tensor = torch.cat([tensor[0:offset + 1], emb[0:emb_len], tensor[offset + 1 + emb_len:]])
diff --git a/modules/sd_hijack_inpainting.py b/modules/sd_hijack_inpainting.py
index 0c37fef89..3405e8a40 100644
--- a/modules/sd_hijack_inpainting.py
+++ b/modules/sd_hijack_inpainting.py
@@ -53,7 +53,7 @@ def p_sample_plms(self, x, c, t, index, repeat_noise=False, use_original_steps=F
 
     def get_x_prev_and_pred_x0(e_t, index):
         # select parameters corresponding to the currently considered timestep
-        print(alphas[index]) # DML ISSUE: PLMS Sampling does not work without this print.
+        print(alphas[index]) # DML Solution: PLMS Sampling does not work without this print.
         a_t = torch.full((b, 1, 1, 1), alphas[index], device=device)
         a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device)
         sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device)
diff --git a/setup.py b/setup.py
index aa01a3de5..237fd22d7 100644
--- a/setup.py
+++ b/setup.py
@@ -254,15 +254,24 @@ def install_repositories():
         return os.path.join(os.path.dirname(__file__), 'repositories', name)
     log.info('Installing repositories')
     os.makedirs(os.path.join(os.path.dirname(__file__), 'repositories'), exist_ok=True)
-    stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") # DML TODO: check samplers work well
-    stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "d4c168b2ad29d82e5fdfea4d598075f40a3b0341")
-    clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit)
+    try:
+        import torch_directml
+        stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/lshqqytiger/stablediffusion-directml.git") # DML Solution: DDIM sampler fix
+        stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "d4c168b2ad29d82e5fdfea4d598075f40a3b0341")
+        clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit)
+        k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/lshqqytiger/k-diffusion-directml.git') # DML Solution: DPM fix
+        k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "47b6ef08bca986ff5e72815e74a419ef6616bdbb")
+        clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit)
+    except:
+        stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git")
+        stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf")
+        clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit)
+        k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git')
+        k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919")
+        clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit)
     taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git")
     taming_transformers_commit = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "3ba01b241669f5ade541ce990f7650a3b8f65318")
     clone(taming_transformers_repo, d('taming-transformers'), taming_transformers_commit)
-    k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git') # DML TODO: check samplers work well
-    k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "47b6ef08bca986ff5e72815e74a419ef6616bdbb")
-    clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit)
     codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git')
     codeformer_commit = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af")
     clone(codeformer_repo, d('CodeFormer'), codeformer_commit)

From cb664cf3328d80c36be89e74f43bdffa6f24de88 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Tue, 25 Apr 2023 02:02:12 +0900
Subject: [PATCH 03/69] Update TODO_DML.

---
 TODO_DML.md | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/TODO_DML.md b/TODO_DML.md
index af6559ac1..f359ba1b3 100644
--- a/TODO_DML.md
+++ b/TODO_DML.md
@@ -20,3 +20,14 @@ What should we do?
 1. Use any fixed value as the available memory capacity.
 2. Use `atiadlxx`(AMD/ATI GPU driver library) to infer vram information as similar as possible to the actual value. (works for AMDGPUs)
 3. or another better way.
+
+- Half mode does not work well.
+
+In half precision, we get an error like `mat1 and mat2 must have the same dtype`.
+
+I solved this problem by overriding forward of GroupNorm / LayerNorm / Conv2d / Linear to convert input tensor to full precision (and convert to float16 before return).
+
+What should we do?
+
+1. Override forwards (same solution)
+2. When using DirectML, set the default to full precision and notify the user.

From 9dc8581ba04cd092885674ffe6681490f1238532 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Tue, 25 Apr 2023 21:22:47 +0900
Subject: [PATCH 04/69] Add hijack for DirectML. Unstable & Under testing.

---
 modules/sd_hijack_directml.py | 170 ++++++++++++++++++++++++++++++++++
 modules/shared.py             |   4 +
 setup.py                      |  21 ++---
 3 files changed, 180 insertions(+), 15 deletions(-)
 create mode 100644 modules/sd_hijack_directml.py

diff --git a/modules/sd_hijack_directml.py b/modules/sd_hijack_directml.py
new file mode 100644
index 000000000..e2e6fa50f
--- /dev/null
+++ b/modules/sd_hijack_directml.py
@@ -0,0 +1,170 @@
+import torch
+from tqdm.auto import tqdm
+
+from shared import device
+
+# k-diffusion
+from k_diffusion import sampling
+
+def dpm_solver_adaptive(self, x, t_start, t_end, order=3, rtol=0.05, atol=0.0078, h_init=0.05, pcoeff=0., icoeff=1., dcoeff=0., accept_safety=0.81, eta=0., s_noise=1., noise_sampler=None):
+    noise_sampler = sampling.default_noise_sampler(x) if noise_sampler is None else noise_sampler
+    if order not in {2, 3}:
+        raise ValueError('order should be 2 or 3')
+    forward = t_end > t_start
+    if not forward and eta:
+        raise ValueError('eta must be 0 for reverse sampling')
+    h_init = abs(h_init) * (1 if forward else -1)
+    atol = torch.tensor(atol).to(device)
+    rtol = torch.tensor(rtol).to(device)
+    s = t_start
+    x_prev = x
+    accept = True
+    pid = sampling.PIDStepSizeController(h_init, pcoeff, icoeff, dcoeff, 1.5 if eta else order, accept_safety)
+    info = {'steps': 0, 'nfe': 0, 'n_accept': 0, 'n_reject': 0}
+
+    while s < t_end - 1e-5 if forward else s > t_end + 1e-5:
+        eps_cache = {}
+        t = torch.minimum(t_end, s + pid.h) if forward else torch.maximum(t_end, s + pid.h)
+        if eta:
+            sd, su = sampling.get_ancestral_step(self.sigma(s), self.sigma(t), eta)
+            t_ = torch.minimum(t_end, self.t(sd))
+            su = (self.sigma(t) ** 2 - self.sigma(t_) ** 2) ** 0.5
+        else:
+            t_, su = t, 0.
+
+        eps, eps_cache = self.eps(eps_cache, 'eps', x, s)
+        denoised = x - self.sigma(s) * eps
+
+        if order == 2:
+            x_low, eps_cache = self.dpm_solver_1_step(x, s, t_, eps_cache=eps_cache)
+            x_high, eps_cache = self.dpm_solver_2_step(x, s, t_, eps_cache=eps_cache)
+        else:
+            x_low, eps_cache = self.dpm_solver_2_step(x, s, t_, r1=1 / 3, eps_cache=eps_cache)
+            x_high, eps_cache = self.dpm_solver_3_step(x, s, t_, eps_cache=eps_cache)
+        delta = torch.maximum(atol, rtol * torch.maximum(x_low.abs(), x_prev.abs()))
+        error = torch.linalg.norm((x_low - x_high) / delta) / x.numel() ** 0.5
+        accept = pid.propose_step(error)
+        if accept:
+            x_prev = x_low
+            x = x_high + su * s_noise * noise_sampler(self.sigma(s), self.sigma(t))
+            s = t
+            info['n_accept'] += 1
+        else:
+            info['n_reject'] += 1
+        info['nfe'] += order
+        info['steps'] += 1
+
+        if self.info_callback is not None:
+            self.info_callback({'x': x, 'i': info['steps'] - 1, 't': s, 't_up': s, 'denoised': denoised, 'error': error, 'h': pid.h, **info})
+
+    return x, info
+
+
+@torch.no_grad()
+def sample_dpm_fast(model, x, sigma_min, sigma_max, n, extra_args=None, callback=None, disable=None, eta=0., s_noise=1., noise_sampler=None):
+    """DPM-Solver-Fast (fixed step size). See https://arxiv.org/abs/2206.00927."""
+    if sigma_min <= 0 or sigma_max <= 0:
+        raise ValueError('sigma_min and sigma_max must not be 0')
+    with tqdm(total=n, disable=disable) as pbar:
+        dpm_solver = sampling.DPMSolver(model, extra_args, eps_callback=pbar.update)
+        if callback is not None:
+            dpm_solver.info_callback = lambda info: callback({'sigma': dpm_solver.sigma(info['t']), 'sigma_hat': dpm_solver.sigma(info['t_up']), **info})
+        return dpm_solver.dpm_solver_fast(x, dpm_solver.t(torch.tensor(sigma_max).to(device)), dpm_solver.t(torch.tensor(sigma_min).to(device)), n, eta, s_noise, noise_sampler)
+
+
+@torch.no_grad()
+def sample_dpm_adaptive(model, x, sigma_min, sigma_max, extra_args=None, callback=None, disable=None, order=3, rtol=0.05, atol=0.0078, h_init=0.05, pcoeff=0., icoeff=1., dcoeff=0., accept_safety=0.81, eta=0., s_noise=1., noise_sampler=None, return_info=False):
+    """DPM-Solver-12 and 23 (adaptive step size). See https://arxiv.org/abs/2206.00927."""
+    if sigma_min <= 0 or sigma_max <= 0:
+        raise ValueError('sigma_min and sigma_max must not be 0')
+    with tqdm(disable=disable) as pbar:
+        dpm_solver = sampling.DPMSolver(model, extra_args, eps_callback=pbar.update)
+        if callback is not None:
+            dpm_solver.info_callback = lambda info: callback({'sigma': dpm_solver.sigma(info['t']), 'sigma_hat': dpm_solver.sigma(info['t_up']), **info})
+        x, info = dpm_solver.dpm_solver_adaptive(x, dpm_solver.t(torch.tensor(sigma_max).to(device)), dpm_solver.t(torch.tensor(sigma_min).to(device)), order, rtol, atol, h_init, pcoeff, icoeff, dcoeff, accept_safety, eta, s_noise, noise_sampler)
+    if return_info:
+        return x, info
+    return x
+
+sampling.DPMSolver.dpm_solver_adaptive = dpm_solver_adaptive
+sampling.sample_dpm_fast = sample_dpm_fast
+sampling.sample_dpm_adaptive = sample_dpm_adaptive
+
+# stablediffusion
+from ldm.models.diffusion.ddim import DDIMSampler
+
+@torch.no_grad()
+def p_sample_ddim(self, x, c, t, index, repeat_noise=False, use_original_steps=False, quantize_denoised=False,
+                    temperature=1., noise_dropout=0., score_corrector=None, corrector_kwargs=None,
+                    unconditional_guidance_scale=1., unconditional_conditioning=None,
+                    dynamic_threshold=None):
+    b, *_, device = *x.shape, x.device
+
+    if unconditional_conditioning is None or unconditional_guidance_scale == 1.:
+        model_output = self.model.apply_model(x, t, c)
+    else:
+        x_in = torch.cat([x] * 2)
+        t_in = torch.cat([t] * 2)
+        if isinstance(c, dict):
+            assert isinstance(unconditional_conditioning, dict)
+            c_in = dict()
+            for k in c:
+                if isinstance(c[k], list):
+                    c_in[k] = [torch.cat([
+                        unconditional_conditioning[k][i],
+                        c[k][i]]) for i in range(len(c[k]))]
+                else:
+                    c_in[k] = torch.cat([
+                            unconditional_conditioning[k],
+                            c[k]])
+        elif isinstance(c, list):
+            c_in = list()
+            assert isinstance(unconditional_conditioning, list)
+            for i in range(len(c)):
+                c_in.append(torch.cat([unconditional_conditioning[i], c[i]]))
+        else:
+            c_in = torch.cat([unconditional_conditioning, c])
+        model_uncond, model_t = self.model.apply_model(x_in, t_in, c_in).chunk(2)
+        model_output = model_uncond + unconditional_guidance_scale * (model_t - model_uncond)
+
+    if self.model.parameterization == "v":
+        e_t = self.model.predict_eps_from_z_and_v(x, t, model_output)
+    else:
+        e_t = model_output
+
+    if score_corrector is not None:
+        assert self.model.parameterization == "eps", 'not implemented'
+        e_t = score_corrector.modify_score(self.model, e_t, x, t, c, **corrector_kwargs)
+
+    alphas = self.model.alphas_cumprod if use_original_steps else self.ddim_alphas
+    alphas_prev = self.model.alphas_cumprod_prev if use_original_steps else self.ddim_alphas_prev
+    sqrt_one_minus_alphas = self.model.sqrt_one_minus_alphas_cumprod if use_original_steps else self.ddim_sqrt_one_minus_alphas
+    sigmas = self.model.ddim_sigmas_for_original_num_steps if use_original_steps else self.ddim_sigmas
+    # select parameters corresponding to the currently considered timestep
+    print(alphas[index]) # DML Solution: DDIM Sampling does not work without this print.
+    a_t = torch.full((b, 1, 1, 1), alphas[index], device=device)
+    a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device)
+    sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device)
+    sqrt_one_minus_at = torch.full((b, 1, 1, 1), sqrt_one_minus_alphas[index],device=device)
+
+    # current prediction for x_0
+    if self.model.parameterization != "v":
+        pred_x0 = (x - sqrt_one_minus_at * e_t) / a_t.sqrt()
+    else:
+        pred_x0 = self.model.predict_start_from_z_and_v(x, t, model_output)
+
+    if quantize_denoised:
+        pred_x0, _, *_ = self.model.first_stage_model.quantize(pred_x0)
+
+    if dynamic_threshold is not None:
+        raise NotImplementedError()
+
+    # direction pointing to x_t
+    dir_xt = (1. - a_prev - sigma_t**2).sqrt() * e_t
+    noise = sigma_t * noise_like(x.shape, device, repeat_noise) * temperature
+    if noise_dropout > 0.:
+        noise = torch.nn.functional.dropout(noise, p=noise_dropout)
+    x_prev = a_prev.sqrt() * pred_x0 + dir_xt + noise
+    return x_prev, pred_x0
+
+DDIMSampler.p_sample_ddim = p_sample_ddim
diff --git a/modules/shared.py b/modules/shared.py
index f147317cd..6267a4a83 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -62,6 +62,10 @@ sd_model = None
 clip_model = None
 
 
+#if device.type == 'privateuseone':
+#    import sd_hijack_directml
+
+
 def reload_hypernetworks():
     from modules.hypernetworks import hypernetwork
     global hypernetworks # pylint: disable=W0603
diff --git a/setup.py b/setup.py
index 237fd22d7..292529300 100644
--- a/setup.py
+++ b/setup.py
@@ -254,21 +254,12 @@ def install_repositories():
         return os.path.join(os.path.dirname(__file__), 'repositories', name)
     log.info('Installing repositories')
     os.makedirs(os.path.join(os.path.dirname(__file__), 'repositories'), exist_ok=True)
-    try:
-        import torch_directml
-        stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/lshqqytiger/stablediffusion-directml.git") # DML Solution: DDIM sampler fix
-        stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "d4c168b2ad29d82e5fdfea4d598075f40a3b0341")
-        clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit)
-        k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/lshqqytiger/k-diffusion-directml.git') # DML Solution: DPM fix
-        k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "47b6ef08bca986ff5e72815e74a419ef6616bdbb")
-        clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit)
-    except:
-        stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git")
-        stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf")
-        clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit)
-        k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git')
-        k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919")
-        clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit)
+    stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git")
+    stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf")
+    clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit)
+    k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git')
+    k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919")
+    clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit)
     taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git")
     taming_transformers_commit = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "3ba01b241669f5ade541ce990f7650a3b8f65318")
     clone(taming_transformers_repo, d('taming-transformers'), taming_transformers_commit)

From db56da075a2740bac6ba93c9eec621db1238ac62 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Tue, 25 Apr 2023 23:04:52 +0900
Subject: [PATCH 05/69] need full precision for model & vae. Stable & tested.

---
 modules/sd_hijack_directml.py      | 11 +++++++++++
 modules/sd_hijack_optimizations.py |  4 ++--
 modules/shared.py                  | 12 +++++++-----
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/modules/sd_hijack_directml.py b/modules/sd_hijack_directml.py
index e2e6fa50f..b640fc848 100644
--- a/modules/sd_hijack_directml.py
+++ b/modules/sd_hijack_directml.py
@@ -92,6 +92,7 @@ sampling.sample_dpm_adaptive = sample_dpm_adaptive
 
 # stablediffusion
 from ldm.models.diffusion.ddim import DDIMSampler
+from ldm.modules.diffusionmodules.util import noise_like
 
 @torch.no_grad()
 def p_sample_ddim(self, x, c, t, index, repeat_noise=False, use_original_steps=False, quantize_denoised=False,
@@ -168,3 +169,13 @@ def p_sample_ddim(self, x, c, t, index, repeat_noise=False, use_original_steps=F
     return x_prev, pred_x0
 
 DDIMSampler.p_sample_ddim = p_sample_ddim
+
+# torch
+
+Generator_init = torch.Generator.__init__
+def Generator_init_fix(self, device = None, *args, **kwargs):
+    if device is not None and device.type == 'privateuseone':
+        return Generator_init(self, 'cpu', *args, **kwargs) # DML Solution: torch.Generator fallback to cpu.
+    else:
+        return Generator_init(self, device, *args, **kwargs)
+torch.Generator.__init__ = Generator_init_fix
diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py
index 0f1f2a757..7e1ba79b3 100644
--- a/modules/sd_hijack_optimizations.py
+++ b/modules/sd_hijack_optimizations.py
@@ -32,7 +32,7 @@ def get_available_vram():
         return mem_free_total
     elif shared.device.type == 'privateuseone':
         # DML ISSUE: There's no way to get any memory info.
-        return 1048576
+        return 1073741824
     else:
         return psutil.virtual_memory().available
 
@@ -200,7 +200,7 @@ def einsum_op_cuda(q, k, v):
 
 def einsum_op_dml(q, k, v):
     # DML ISSUE: There's no way to get any memory info.
-    return einsum_op_tensor_mem(q, k, v, 1024)
+    return einsum_op_tensor_mem(q, k, v, 1073741824)
 
 def einsum_op(q, k, v):
     if q.device.type == 'cuda':
diff --git a/modules/shared.py b/modules/shared.py
index 6267a4a83..9aa4e8d64 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -57,13 +57,15 @@ devices.device, devices.device_interrogate, devices.device_gfpgan, devices.devic
     (devices.cpu if any(y in cmd_opts.use_cpu for y in [x, 'all']) else devices.get_optimal_device() for x in ['sd', 'interrogate', 'gfpgan', 'esrgan', 'codeformer'])
 
 device = devices.device
+is_device_dml = False
 sd_upscalers = []
 sd_model = None
 clip_model = None
 
 
-#if device.type == 'privateuseone':
-#    import sd_hijack_directml
+if device.type == 'privateuseone':
+    import sd_hijack_directml
+    is_device_dml = True
 
 
 def reload_hypernetworks():
@@ -251,7 +253,7 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), {
     "comma_padding_backtrack": OptionInfo(20, "Increase coherency by padding from the last comma within n tokens when using more than 75 tokens", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }),
     "CLIP_stop_at_last_layers": OptionInfo(1, "Clip skip", gr.Slider, {"minimum": 1, "maximum": 12, "step": 1}),
     "upcast_attn": OptionInfo(False, "Upcast cross attention layer to float32"),
-    "cross_attention_optimization": OptionInfo("Scaled-Dot-Product", "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }),
+    "cross_attention_optimization": OptionInfo("Sub-quadratic" if is_device_dml else "Scaled-Dot-Product", "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }),
     "cross_attention_options": OptionInfo([], "Cross-attention advanced options", gr.CheckboxGroup, lambda: {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention']}),
     "sub_quad_q_chunk_size": OptionInfo(512, "Sub-quadratic cross-attention query chunk size for the  layer optimization to use", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8}),
     "sub_quad_kv_chunk_size": OptionInfo(512, "Sub-quadratic cross-attentionkv chunk size for the sub-quadratic cross-attention layer optimization to use", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8}),
@@ -328,8 +330,8 @@ options_templates.update(options_section(('saving-paths', "Image Paths"), {
 options_templates.update(options_section(('cuda', "CUDA Settings"), {
     "precision": OptionInfo("Autocast", "Precision type", gr.Radio, lambda: {"choices": ["Autocast", "Full"]}),
     "cuda_dtype": OptionInfo("FP16", "Device precision type", gr.Radio, lambda: {"choices": ["FP32", "FP16", "BF16"]}),
-    "no_half": OptionInfo(False, "Use full precision for model (--no-half)"),
-    "no_half_vae": OptionInfo(False, "Use full precision for VAE (--no-half-vae)"),
+    "no_half": OptionInfo(True if is_device_dml else False, "Use full precision for model (--no-half)"),
+    "no_half_vae": OptionInfo(True if is_device_dml else False, "Use full precision for VAE (--no-half-vae)"),
     "disable_nan_check": OptionInfo(True, "Do not check if produced images/latent spaces have NaN values"),
     "opt_channelslast": OptionInfo(False, "Use channels last as torch memory format "),
     "cudnn_benchmark": OptionInfo(False, "Enable cuDNN benchmark feature"),

From 53736ea7cbb7c08dd976cc8deff61197319fe492 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Tue, 25 Apr 2023 23:07:25 +0900
Subject: [PATCH 06/69] fix

---
 modules/shared.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/shared.py b/modules/shared.py
index 9aa4e8d64..27018c5a7 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -64,7 +64,7 @@ clip_model = None
 
 
 if device.type == 'privateuseone':
-    import sd_hijack_directml
+    import modules.sd_hijack_directml
     is_device_dml = True
 
 

From 32634298d7ff047f53fb4d17d28068969c10e285 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Tue, 25 Apr 2023 23:09:01 +0900
Subject: [PATCH 07/69] fix

---
 modules/sd_hijack_directml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/sd_hijack_directml.py b/modules/sd_hijack_directml.py
index b640fc848..c782921af 100644
--- a/modules/sd_hijack_directml.py
+++ b/modules/sd_hijack_directml.py
@@ -1,7 +1,7 @@
 import torch
 from tqdm.auto import tqdm
 
-from shared import device
+from modules.shared import device
 
 # k-diffusion
 from k_diffusion import sampling

From eb072db23c3e2144a991274f37b68006d2ce7781 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Tue, 25 Apr 2023 23:19:03 +0900
Subject: [PATCH 08/69] Add dml_specific.

---
 modules/{sd_hijack_directml.py => dml_specific.py} | 9 ++-------
 modules/shared.py                                  | 2 +-
 2 files changed, 3 insertions(+), 8 deletions(-)
 rename modules/{sd_hijack_directml.py => dml_specific.py} (95%)

diff --git a/modules/sd_hijack_directml.py b/modules/dml_specific.py
similarity index 95%
rename from modules/sd_hijack_directml.py
rename to modules/dml_specific.py
index c782921af..e8503d745 100644
--- a/modules/sd_hijack_directml.py
+++ b/modules/dml_specific.py
@@ -2,6 +2,7 @@ import torch
 from tqdm.auto import tqdm
 
 from modules.shared import device
+from modules.sd_hijack_utils import CondFunc
 
 # k-diffusion
 from k_diffusion import sampling
@@ -172,10 +173,4 @@ DDIMSampler.p_sample_ddim = p_sample_ddim
 
 # torch
 
-Generator_init = torch.Generator.__init__
-def Generator_init_fix(self, device = None, *args, **kwargs):
-    if device is not None and device.type == 'privateuseone':
-        return Generator_init(self, 'cpu', *args, **kwargs) # DML Solution: torch.Generator fallback to cpu.
-    else:
-        return Generator_init(self, device, *args, **kwargs)
-torch.Generator.__init__ = Generator_init_fix
+CondFunc('torchsde._brownian.brownian_interval._randn', lambda _, size, dtype, device, seed: torch.randn(size, dtype=dtype, device=torch.device("cpu"), generator=torch.Generator(torch.device("cpu")).manual_seed(int(seed))).to(device), lambda _, size, dtype, device, seed: device.type == 'privateuseone')
diff --git a/modules/shared.py b/modules/shared.py
index 27018c5a7..99f4a986b 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -64,7 +64,7 @@ clip_model = None
 
 
 if device.type == 'privateuseone':
-    import modules.sd_hijack_directml
+    import modules.dml_specific
     is_device_dml = True
 
 

From 09ae33cdf741e927f18713ff0c9b1b1d7e0465c2 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Wed, 26 Apr 2023 12:21:44 +0900
Subject: [PATCH 09/69] Implement torch.dml. VERY UNSTABLE & NOT TESTED.

---
 extensions-builtin/sd-webui-controlnet        |  2 +-
 extensions-builtin/seed_travel                |  2 +-
 modules/dml/__init__.py                       | 32 +++++++
 .../{dml_specific.py => dml/kdiffusion.py}    | 89 +------------------
 modules/dml/optimizer/amd/__init__.py         |  7 ++
 modules/dml/optimizer/amd/driver/atiadlxx.py  | 43 +++++++++
 .../dml/optimizer/amd/driver/atiadlxx_apis.py | 50 +++++++++++
 .../optimizer/amd/driver/atiadlxx_defines.py  |  1 +
 .../amd/driver/atiadlxx_structures.py         | 87 ++++++++++++++++++
 modules/dml/optimizer/amd/memory.py           |  0
 modules/dml/optimizer/intel/__init__.py       |  7 ++
 modules/dml/optimizer/nvidia/__init__.py      |  7 ++
 modules/dml/optimizer/optimizer.py            |  8 ++
 modules/dml/optimizer/unknown/__init__.py     |  6 ++
 modules/dml/stablediffusion.py                | 80 +++++++++++++++++
 modules/dml/torch.py                          |  5 ++
 modules/sd_hijack_optimizations.py            | 12 ++-
 modules/shared.py                             |  4 +-
 wiki                                          |  2 +-
 19 files changed, 347 insertions(+), 97 deletions(-)
 create mode 100644 modules/dml/__init__.py
 rename modules/{dml_specific.py => dml/kdiffusion.py} (52%)
 create mode 100644 modules/dml/optimizer/amd/__init__.py
 create mode 100644 modules/dml/optimizer/amd/driver/atiadlxx.py
 create mode 100644 modules/dml/optimizer/amd/driver/atiadlxx_apis.py
 create mode 100644 modules/dml/optimizer/amd/driver/atiadlxx_defines.py
 create mode 100644 modules/dml/optimizer/amd/driver/atiadlxx_structures.py
 create mode 100644 modules/dml/optimizer/amd/memory.py
 create mode 100644 modules/dml/optimizer/intel/__init__.py
 create mode 100644 modules/dml/optimizer/nvidia/__init__.py
 create mode 100644 modules/dml/optimizer/optimizer.py
 create mode 100644 modules/dml/optimizer/unknown/__init__.py
 create mode 100644 modules/dml/stablediffusion.py
 create mode 100644 modules/dml/torch.py

diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index f16c9e522..c5fbfc31d 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit f16c9e5221bcf9490f5fd93ff0d38027a14bc4d6
+Subproject commit c5fbfc31d002ff83bc692e1f24b7b1c9183dbe72
diff --git a/extensions-builtin/seed_travel b/extensions-builtin/seed_travel
index 1a97ebb43..ffe0553c5 160000
--- a/extensions-builtin/seed_travel
+++ b/extensions-builtin/seed_travel
@@ -1 +1 @@
-Subproject commit 1a97ebb43428b43c41001cfe79e50b468126d99f
+Subproject commit ffe0553c59e91067ebf1e4fc7ad85ca9c870bf57
diff --git a/modules/dml/__init__.py b/modules/dml/__init__.py
new file mode 100644
index 000000000..fbda39384
--- /dev/null
+++ b/modules/dml/__init__.py
@@ -0,0 +1,32 @@
+import torch
+import torch_directml
+
+import modules.dml.kdiffusion
+import modules.dml.stablediffusion
+import modules.dml.torch
+
+from optimizer.unknown import UnknownOptimizer
+
+class DirectML():
+    def get_optimizer(self, device: torch.device):
+        assert(device.type == 'privateuseone')
+        try:
+            device_name = torch_directml.device_name(device.index)
+            if 'NVIDIA' in device_name or 'GeForce' in device_name:
+                from optimizer.nvidia import nVidiaOptimizer as optimizer
+            elif 'AMD' in device_name or 'Radeon' in device_name:
+                from optimizer.amd import AMDOptimizer as optimizer
+            elif 'Intel' in device_name:
+                from optimizer.intel import IntelOptimizer as optimizer
+            else:
+                return UnknownOptimizer
+            return optimizer
+        except:
+            return UnknownOptimizer
+
+    def memory_stats(self, device: torch.device):
+        optimizer = self.get_optimizer(device)
+        return optimizer.memory_stats(device.index)
+
+# Alternative of torch.cuda for DirectML.
+torch.dml = DirectML
diff --git a/modules/dml_specific.py b/modules/dml/kdiffusion.py
similarity index 52%
rename from modules/dml_specific.py
rename to modules/dml/kdiffusion.py
index e8503d745..2eced885f 100644
--- a/modules/dml_specific.py
+++ b/modules/dml/kdiffusion.py
@@ -2,9 +2,6 @@ import torch
 from tqdm.auto import tqdm
 
 from modules.shared import device
-from modules.sd_hijack_utils import CondFunc
-
-# k-diffusion
 from k_diffusion import sampling
 
 def dpm_solver_adaptive(self, x, t_start, t_end, order=3, rtol=0.05, atol=0.0078, h_init=0.05, pcoeff=0., icoeff=1., dcoeff=0., accept_safety=0.81, eta=0., s_noise=1., noise_sampler=None):
@@ -89,88 +86,4 @@ def sample_dpm_adaptive(model, x, sigma_min, sigma_max, extra_args=None, callbac
 
 sampling.DPMSolver.dpm_solver_adaptive = dpm_solver_adaptive
 sampling.sample_dpm_fast = sample_dpm_fast
-sampling.sample_dpm_adaptive = sample_dpm_adaptive
-
-# stablediffusion
-from ldm.models.diffusion.ddim import DDIMSampler
-from ldm.modules.diffusionmodules.util import noise_like
-
-@torch.no_grad()
-def p_sample_ddim(self, x, c, t, index, repeat_noise=False, use_original_steps=False, quantize_denoised=False,
-                    temperature=1., noise_dropout=0., score_corrector=None, corrector_kwargs=None,
-                    unconditional_guidance_scale=1., unconditional_conditioning=None,
-                    dynamic_threshold=None):
-    b, *_, device = *x.shape, x.device
-
-    if unconditional_conditioning is None or unconditional_guidance_scale == 1.:
-        model_output = self.model.apply_model(x, t, c)
-    else:
-        x_in = torch.cat([x] * 2)
-        t_in = torch.cat([t] * 2)
-        if isinstance(c, dict):
-            assert isinstance(unconditional_conditioning, dict)
-            c_in = dict()
-            for k in c:
-                if isinstance(c[k], list):
-                    c_in[k] = [torch.cat([
-                        unconditional_conditioning[k][i],
-                        c[k][i]]) for i in range(len(c[k]))]
-                else:
-                    c_in[k] = torch.cat([
-                            unconditional_conditioning[k],
-                            c[k]])
-        elif isinstance(c, list):
-            c_in = list()
-            assert isinstance(unconditional_conditioning, list)
-            for i in range(len(c)):
-                c_in.append(torch.cat([unconditional_conditioning[i], c[i]]))
-        else:
-            c_in = torch.cat([unconditional_conditioning, c])
-        model_uncond, model_t = self.model.apply_model(x_in, t_in, c_in).chunk(2)
-        model_output = model_uncond + unconditional_guidance_scale * (model_t - model_uncond)
-
-    if self.model.parameterization == "v":
-        e_t = self.model.predict_eps_from_z_and_v(x, t, model_output)
-    else:
-        e_t = model_output
-
-    if score_corrector is not None:
-        assert self.model.parameterization == "eps", 'not implemented'
-        e_t = score_corrector.modify_score(self.model, e_t, x, t, c, **corrector_kwargs)
-
-    alphas = self.model.alphas_cumprod if use_original_steps else self.ddim_alphas
-    alphas_prev = self.model.alphas_cumprod_prev if use_original_steps else self.ddim_alphas_prev
-    sqrt_one_minus_alphas = self.model.sqrt_one_minus_alphas_cumprod if use_original_steps else self.ddim_sqrt_one_minus_alphas
-    sigmas = self.model.ddim_sigmas_for_original_num_steps if use_original_steps else self.ddim_sigmas
-    # select parameters corresponding to the currently considered timestep
-    print(alphas[index]) # DML Solution: DDIM Sampling does not work without this print.
-    a_t = torch.full((b, 1, 1, 1), alphas[index], device=device)
-    a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device)
-    sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device)
-    sqrt_one_minus_at = torch.full((b, 1, 1, 1), sqrt_one_minus_alphas[index],device=device)
-
-    # current prediction for x_0
-    if self.model.parameterization != "v":
-        pred_x0 = (x - sqrt_one_minus_at * e_t) / a_t.sqrt()
-    else:
-        pred_x0 = self.model.predict_start_from_z_and_v(x, t, model_output)
-
-    if quantize_denoised:
-        pred_x0, _, *_ = self.model.first_stage_model.quantize(pred_x0)
-
-    if dynamic_threshold is not None:
-        raise NotImplementedError()
-
-    # direction pointing to x_t
-    dir_xt = (1. - a_prev - sigma_t**2).sqrt() * e_t
-    noise = sigma_t * noise_like(x.shape, device, repeat_noise) * temperature
-    if noise_dropout > 0.:
-        noise = torch.nn.functional.dropout(noise, p=noise_dropout)
-    x_prev = a_prev.sqrt() * pred_x0 + dir_xt + noise
-    return x_prev, pred_x0
-
-DDIMSampler.p_sample_ddim = p_sample_ddim
-
-# torch
-
-CondFunc('torchsde._brownian.brownian_interval._randn', lambda _, size, dtype, device, seed: torch.randn(size, dtype=dtype, device=torch.device("cpu"), generator=torch.Generator(torch.device("cpu")).manual_seed(int(seed))).to(device), lambda _, size, dtype, device, seed: device.type == 'privateuseone')
+sampling.sample_dpm_adaptive = sample_dpm_adaptive
\ No newline at end of file
diff --git a/modules/dml/optimizer/amd/__init__.py b/modules/dml/optimizer/amd/__init__.py
new file mode 100644
index 000000000..37ea3ea91
--- /dev/null
+++ b/modules/dml/optimizer/amd/__init__.py
@@ -0,0 +1,7 @@
+from modules.dml.optimizer.optimizer import Optimizer
+from driver.atiadlxx import ATIADLxx
+
+class AMDOptimizer(Optimizer):
+    driver: ATIADLxx = ATIADLxx()
+    def memory_stats(self, index):
+        return (AMDOptimizer.driver.iHyperMemorySize, AMDOptimizer.driver.get_dedicated_vram_usage(index))
diff --git a/modules/dml/optimizer/amd/driver/atiadlxx.py b/modules/dml/optimizer/amd/driver/atiadlxx.py
new file mode 100644
index 000000000..abb5eca96
--- /dev/null
+++ b/modules/dml/optimizer/amd/driver/atiadlxx.py
@@ -0,0 +1,43 @@
+import ctypes as C
+from .atiadlxx_apis import *
+from .atiadlxx_structures import *
+from .atiadlxx_defines import *
+
+class ATIADLxx(object):
+    iHyperMemorySize = 0
+
+    def __init__(self):
+        self.context = ADL_CONTEXT_HANDLE()
+        ADL2_Main_Control_Create(ADL_Main_Memory_Alloc, 1, C.byref(self.context))
+        num_adapters = C.c_int(-1)
+        ADL2_Adapter_NumberOfAdapters_Get(self.context, C.byref(num_adapters))
+        AdapterInfoArray = (AdapterInfo * num_adapters.value)()
+        ADL2_Adapter_AdapterInfo_Get(self.context, C.cast(AdapterInfoArray, LPAdapterInfo), C.sizeof(AdapterInfoArray))
+        self.devices = []
+        for adapter in AdapterInfoArray:
+            self.devices.append(adapter)
+        self.iHyperMemorySize = self.get_memory_info2(0).iHyperMemorySize
+
+    def get_memory_info2(self, adapterIndex: int) -> ADLMemoryInfo2:
+        info = ADLMemoryInfo2()
+
+        if ADL2_Adapter_MemoryInfo2_Get(self.context, adapterIndex, C.byref(info)) != ADL_OK:
+            raise RuntimeError("ADL2: Failed to get MemoryInfo2")
+        
+        return info
+
+    def get_dedicated_vram_usage(self, index: int) -> int:
+        usage = C.c_int(-1)
+
+        if ADL2_Adapter_DedicatedVRAMUsage_Get(self.context, self.devices[index].iAdapterIndex, C.byref(usage)) != ADL_OK:
+            raise RuntimeError("ADL2: Failed to get DedicatedVRAMUsage")
+
+        return usage.value
+
+    def get_vram_usage(self, index: int) -> int:
+        usage = C.c_int(-1)
+
+        if ADL2_Adapter_VRAMUsage_Get(self.context, self.devices[index].iAdapterIndex, C.byref(usage)) != ADL_OK:
+            raise RuntimeError("ADL2: Failed to get VRAMUsage")
+
+        return usage.value
diff --git a/modules/dml/optimizer/amd/driver/atiadlxx_apis.py b/modules/dml/optimizer/amd/driver/atiadlxx_apis.py
new file mode 100644
index 000000000..23b7da3b1
--- /dev/null
+++ b/modules/dml/optimizer/amd/driver/atiadlxx_apis.py
@@ -0,0 +1,50 @@
+import ctypes as C
+import platform
+from .atiadlxx_structures import *
+
+_platform = platform.system()
+
+try:
+    if _platform == "Windows":
+        atiadlxx = C.WinDLL("atiadlxx.dll")
+
+        ADL_MAIN_MALLOC_CALLBACK = C.CFUNCTYPE(C.c_void_p, C.c_int)
+        ADL_MAIN_FREE_CALLBACK = C.CFUNCTYPE(None, C.POINTER(C.c_void_p))
+
+        @ADL_MAIN_MALLOC_CALLBACK
+        def ADL_Main_Memory_Alloc(iSize):
+            return C._malloc(iSize)
+
+        @ADL_MAIN_FREE_CALLBACK
+        def ADL_Main_Memory_Free(lpBuffer):
+            if lpBuffer[0] is not None:
+                C._free(lpBuffer[0])
+                lpBuffer[0] = None
+
+        ADL2_Main_Control_Create = atiadlxx.ADL2_Main_Control_Create
+        ADL2_Main_Control_Create.restype = C.c_int
+        ADL2_Main_Control_Create.argtypes = [ADL_MAIN_MALLOC_CALLBACK, C.c_int, ADL_CONTEXT_HANDLE]
+
+        ADL2_Adapter_NumberOfAdapters_Get = atiadlxx.ADL2_Adapter_NumberOfAdapters_Get
+        ADL2_Adapter_NumberOfAdapters_Get.restype = C.c_int
+        ADL2_Adapter_NumberOfAdapters_Get.argtypes = [ADL_CONTEXT_HANDLE, C.POINTER(C.c_int)]
+
+        ADL2_Adapter_AdapterInfo_Get = atiadlxx.ADL2_Adapter_AdapterInfo_Get
+        ADL2_Adapter_AdapterInfo_Get.restype = C.c_int
+        ADL2_Adapter_AdapterInfo_Get.argtypes = [ADL_CONTEXT_HANDLE, LPAdapterInfo, C.c_int]
+
+        ADL2_Adapter_MemoryInfo2_Get = atiadlxx.ADL2_Adapter_MemoryInfo2_Get
+        ADL2_Adapter_MemoryInfo2_Get.restype = C.c_int
+        ADL2_Adapter_MemoryInfo2_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(ADLMemoryInfo2)]
+
+        ADL2_Adapter_DedicatedVRAMUsage_Get = atiadlxx.ADL2_Adapter_DedicatedVRAMUsage_Get
+        ADL2_Adapter_DedicatedVRAMUsage_Get.restype = C.c_int
+        ADL2_Adapter_DedicatedVRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)]
+
+        ADL2_Adapter_VRAMUsage_Get = atiadlxx.ADL2_Adapter_VRAMUsage_Get
+        ADL2_Adapter_VRAMUsage_Get.restype = C.c_int
+        ADL2_Adapter_VRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)]
+    else:
+        print("Warning: experimental graphic memory optimization for AMDGPU is disabled. Because this is not Windows platform.")
+except FileNotFoundError:
+    print("Warning: memory optimization for AMDGPU is disabled. Because couldn't find 'atiadlxx.dll'. Please install GPU driver downloaded from AMD.com.")
\ No newline at end of file
diff --git a/modules/dml/optimizer/amd/driver/atiadlxx_defines.py b/modules/dml/optimizer/amd/driver/atiadlxx_defines.py
new file mode 100644
index 000000000..c242b9819
--- /dev/null
+++ b/modules/dml/optimizer/amd/driver/atiadlxx_defines.py
@@ -0,0 +1 @@
+ADL_OK = 0
\ No newline at end of file
diff --git a/modules/dml/optimizer/amd/driver/atiadlxx_structures.py b/modules/dml/optimizer/amd/driver/atiadlxx_structures.py
new file mode 100644
index 000000000..a68392ec4
--- /dev/null
+++ b/modules/dml/optimizer/amd/driver/atiadlxx_structures.py
@@ -0,0 +1,87 @@
+import ctypes as C
+
+class _ADLPMActivity(C.Structure):
+    __slot__ = [
+        'iActivityPercent',
+        'iCurrentBusLanes',
+        'iCurrentBusSpeed',
+        'iCurrentPerformanceLevel',
+        'iEngineClock',
+        'iMaximumBusLanes',
+        'iMemoryClock',
+        'iReserved',
+        'iSize',
+        'iVddc',
+    ]
+_ADLPMActivity._fields_ = [
+    ('iActivityPercent', C.c_int),
+    ('iCurrentBusLanes', C.c_int),
+    ('iCurrentBusSpeed', C.c_int),
+    ('iCurrentPerformanceLevel', C.c_int),
+    ('iEngineClock', C.c_int),
+    ('iMaximumBusLanes', C.c_int),
+    ('iMemoryClock', C.c_int),
+    ('iReserved', C.c_int),
+    ('iSize', C.c_int),
+    ('iVddc', C.c_int),
+]
+ADLPMActivity = _ADLPMActivity
+
+class _ADLMemoryInfo2(C.Structure):
+    __slot__ = [
+        'iHyperMemorySize',
+        'iInvisibleMemorySize',
+        'iMemoryBandwidth',
+        'iMemorySize',
+        'iVisibleMemorySize',
+        'strMemoryType'
+    ]
+_ADLMemoryInfo2._fields_ = [
+    ('iHyperMemorySize', C.c_longlong),
+    ('iInvisibleMemorySize', C.c_longlong),
+    ('iMemoryBandwidth', C.c_longlong),
+    ('iMemorySize', C.c_longlong),
+    ('iVisibleMemorySize', C.c_longlong),
+    ('strMemoryType', C.c_char * 256)
+]
+ADLMemoryInfo2 = _ADLMemoryInfo2
+
+class _AdapterInfo(C.Structure):
+    __slot__ = [
+        'iSize',
+        'iAdapterIndex',
+        'strUDID',
+        'iBusNumber',
+        'iDeviceNumber',
+        'iFunctionNumber',
+        'iVendorID',
+        'strAdapterName',
+        'strDisplayName',
+        'iPresent',
+        'iExist',
+        'strDriverPath',
+        'strDriverPathExt',
+        'strPNPString',
+        'iOSDisplayIndex',
+    ]
+_AdapterInfo._fields_ = [
+    ('iSize', C.c_int),
+    ('iAdapterIndex', C.c_int),
+    ('strUDID', C.c_char * 256),
+    ('iBusNumber', C.c_int),
+    ('iDeviceNumber', C.c_int),
+    ('iFunctionNumber', C.c_int),
+    ('iVendorID', C.c_int),
+    ('strAdapterName', C.c_char * 256),
+    ('strDisplayName', C.c_char * 256),
+    ('iPresent', C.c_int),
+    ('iExist', C.c_int),
+    ('strDriverPath', C.c_char * 256),
+    ('strDriverPathExt', C.c_char * 256),
+    ('strPNPString', C.c_char * 256),
+    ('iOSDisplayIndex', C.c_int)
+]
+AdapterInfo = _AdapterInfo
+LPAdapterInfo = C.POINTER(_AdapterInfo)
+
+ADL_CONTEXT_HANDLE = C.c_void_p
\ No newline at end of file
diff --git a/modules/dml/optimizer/amd/memory.py b/modules/dml/optimizer/amd/memory.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/modules/dml/optimizer/intel/__init__.py b/modules/dml/optimizer/intel/__init__.py
new file mode 100644
index 000000000..d17cd59d2
--- /dev/null
+++ b/modules/dml/optimizer/intel/__init__.py
@@ -0,0 +1,7 @@
+from modules.dml.optimizer.optimizer import Optimizer
+
+class IntelOptimizer(Optimizer):
+    def memory_stats():
+        raise NotImplementedError()
+        # DML TODO: Implement
+        return
diff --git a/modules/dml/optimizer/nvidia/__init__.py b/modules/dml/optimizer/nvidia/__init__.py
new file mode 100644
index 000000000..d467e8ae5
--- /dev/null
+++ b/modules/dml/optimizer/nvidia/__init__.py
@@ -0,0 +1,7 @@
+from modules.dml.optimizer.optimizer import Optimizer
+
+class nVidiaOptimizer(Optimizer):
+    def memory_stats():
+        raise NotImplementedError()
+        # DML TODO: Implement
+        return
diff --git a/modules/dml/optimizer/optimizer.py b/modules/dml/optimizer/optimizer.py
new file mode 100644
index 000000000..2b2d9ed64
--- /dev/null
+++ b/modules/dml/optimizer/optimizer.py
@@ -0,0 +1,8 @@
+from abc import *
+from typing import *
+
+class Optimizer(metaclass=ABCMeta):
+    driver: Any = None
+    @abstractmethod
+    def memory_stats(self, index: int) -> Tuple[int, int]:
+        pass
diff --git a/modules/dml/optimizer/unknown/__init__.py b/modules/dml/optimizer/unknown/__init__.py
new file mode 100644
index 000000000..f2fbc2ae2
--- /dev/null
+++ b/modules/dml/optimizer/unknown/__init__.py
@@ -0,0 +1,6 @@
+from modules.dml.optimizer.optimizer import Optimizer
+
+class UnknownOptimizer(Optimizer):
+    def memory_stats():
+        # DML TODO: Implement
+        return (1073741824, 0)
diff --git a/modules/dml/stablediffusion.py b/modules/dml/stablediffusion.py
new file mode 100644
index 000000000..cbc4b85fb
--- /dev/null
+++ b/modules/dml/stablediffusion.py
@@ -0,0 +1,80 @@
+import torch
+
+from ldm.models.diffusion.ddim import DDIMSampler
+from ldm.modules.diffusionmodules.util import noise_like
+
+@torch.no_grad()
+def p_sample_ddim(self, x, c, t, index, repeat_noise=False, use_original_steps=False, quantize_denoised=False,
+                    temperature=1., noise_dropout=0., score_corrector=None, corrector_kwargs=None,
+                    unconditional_guidance_scale=1., unconditional_conditioning=None,
+                    dynamic_threshold=None):
+    b, *_, device = *x.shape, x.device
+
+    if unconditional_conditioning is None or unconditional_guidance_scale == 1.:
+        model_output = self.model.apply_model(x, t, c)
+    else:
+        x_in = torch.cat([x] * 2)
+        t_in = torch.cat([t] * 2)
+        if isinstance(c, dict):
+            assert isinstance(unconditional_conditioning, dict)
+            c_in = dict()
+            for k in c:
+                if isinstance(c[k], list):
+                    c_in[k] = [torch.cat([
+                        unconditional_conditioning[k][i],
+                        c[k][i]]) for i in range(len(c[k]))]
+                else:
+                    c_in[k] = torch.cat([
+                            unconditional_conditioning[k],
+                            c[k]])
+        elif isinstance(c, list):
+            c_in = list()
+            assert isinstance(unconditional_conditioning, list)
+            for i in range(len(c)):
+                c_in.append(torch.cat([unconditional_conditioning[i], c[i]]))
+        else:
+            c_in = torch.cat([unconditional_conditioning, c])
+        model_uncond, model_t = self.model.apply_model(x_in, t_in, c_in).chunk(2)
+        model_output = model_uncond + unconditional_guidance_scale * (model_t - model_uncond)
+
+    if self.model.parameterization == "v":
+        e_t = self.model.predict_eps_from_z_and_v(x, t, model_output)
+    else:
+        e_t = model_output
+
+    if score_corrector is not None:
+        assert self.model.parameterization == "eps", 'not implemented'
+        e_t = score_corrector.modify_score(self.model, e_t, x, t, c, **corrector_kwargs)
+
+    alphas = self.model.alphas_cumprod if use_original_steps else self.ddim_alphas
+    alphas_prev = self.model.alphas_cumprod_prev if use_original_steps else self.ddim_alphas_prev
+    sqrt_one_minus_alphas = self.model.sqrt_one_minus_alphas_cumprod if use_original_steps else self.ddim_sqrt_one_minus_alphas
+    sigmas = self.model.ddim_sigmas_for_original_num_steps if use_original_steps else self.ddim_sigmas
+    # select parameters corresponding to the currently considered timestep
+    print(alphas[index]) # DML Solution: DDIM Sampling does not work without this print.
+    a_t = torch.full((b, 1, 1, 1), alphas[index], device=device)
+    a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device)
+    sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device)
+    sqrt_one_minus_at = torch.full((b, 1, 1, 1), sqrt_one_minus_alphas[index],device=device)
+
+    # current prediction for x_0
+    if self.model.parameterization != "v":
+        pred_x0 = (x - sqrt_one_minus_at * e_t) / a_t.sqrt()
+    else:
+        pred_x0 = self.model.predict_start_from_z_and_v(x, t, model_output)
+
+    if quantize_denoised:
+        pred_x0, _, *_ = self.model.first_stage_model.quantize(pred_x0)
+
+    if dynamic_threshold is not None:
+        raise NotImplementedError()
+
+    # direction pointing to x_t
+    dir_xt = (1. - a_prev - sigma_t**2).sqrt() * e_t
+    noise = sigma_t * noise_like(x.shape, device, repeat_noise) * temperature
+    if noise_dropout > 0.:
+        noise = torch.nn.functional.dropout(noise, p=noise_dropout)
+    x_prev = a_prev.sqrt() * pred_x0 + dir_xt + noise
+    return x_prev, pred_x0
+
+DDIMSampler.p_sample_ddim = p_sample_ddim
diff --git a/modules/dml/torch.py b/modules/dml/torch.py
new file mode 100644
index 000000000..04e777258
--- /dev/null
+++ b/modules/dml/torch.py
@@ -0,0 +1,5 @@
+import torch
+
+from modules.sd_hijack_utils import CondFunc
+
+CondFunc('torchsde._brownian.brownian_interval._randn', lambda _, size, dtype, device, seed: torch.randn(size, dtype=dtype, device=torch.device("cpu"), generator=torch.Generator(torch.device("cpu")).manual_seed(int(seed))).to(device), lambda _, size, dtype, device, seed: device.type == 'privateuseone')
diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py
index 7e1ba79b3..79b8e3b47 100644
--- a/modules/sd_hijack_optimizations.py
+++ b/modules/sd_hijack_optimizations.py
@@ -20,6 +20,9 @@ if shared.opts.cross_attention_optimization == "xFormers":
     except Exception:
         pass
 
+if shared.device.type == 'privateuseone':
+    import dml
+
 
 def get_available_vram():
     if shared.device.type == 'cuda':
@@ -31,8 +34,8 @@ def get_available_vram():
         mem_free_total = mem_free_cuda + mem_free_torch
         return mem_free_total
     elif shared.device.type == 'privateuseone':
-        # DML ISSUE: There's no way to get any memory info.
-        return 1073741824
+        mem_total, mem_active = torch.dml.memory_stats(shared.device)
+        return mem_total - mem_active * (1 << 20)
     else:
         return psutil.virtual_memory().available
 
@@ -199,8 +202,9 @@ def einsum_op_cuda(q, k, v):
     return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20))
 
 def einsum_op_dml(q, k, v):
-    # DML ISSUE: There's no way to get any memory info.
-    return einsum_op_tensor_mem(q, k, v, 1073741824)
+    mem_total, mem_active = devices.adl.memory_stats()
+    mem_reserved = mem_total / (1 << 20) * 0.7
+    return einsum_op_tensor_mem(q, k, v, (mem_reserved - mem_active) if mem_reserved > mem_active else 1)
 
 def einsum_op(q, k, v):
     if q.device.type == 'cuda':
diff --git a/modules/shared.py b/modules/shared.py
index 34748c0b9..08ff22cfa 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -64,7 +64,7 @@ clip_model = None
 
 
 if device.type == 'privateuseone':
-    import modules.dml_specific
+    import modules.dml
     is_device_dml = True
 
 
@@ -429,7 +429,7 @@ options_templates.update(options_section(('ui', "Live previews"), {
     "live_previews_enable": OptionInfo(True, "Show live previews of the created image"),
     "show_progress_grid": OptionInfo(True, "Show previews of all images generated in a batch as a grid"),
     "show_progress_every_n_steps": OptionInfo(1, "Show new live preview image every N sampling steps. Set to -1 to show after completion of batch.", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}),
-    "show_progress_type": OptionInfo("Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}), # DML ISSUE: Approx NN does not work well on DirectML device.
+    "show_progress_type": OptionInfo("Approx cheap" if is_device_dml else "Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}), # DML Solution: Use Approx cheap instead of Approx NN as a default progress type.
     "live_preview_content": OptionInfo("Combined", "Live preview subject", gr.Radio, {"choices": ["Combined", "Prompt", "Negative prompt"]}),
     "live_preview_refresh_period": OptionInfo(250, "Progressbar/preview update period, in milliseconds")
 }))
diff --git a/wiki b/wiki
index 066ea609f..12603bcde 160000
--- a/wiki
+++ b/wiki
@@ -1 +1 @@
-Subproject commit 066ea609f6a2630bceb07c679142d83a41a6db0f
+Subproject commit 12603bcdec55df780b18612d58b6d0dcd4c27f96

From 8b75033a111e41d185cbef782b60520559a20e20 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Wed, 26 Apr 2023 12:34:27 +0900
Subject: [PATCH 10/69] fix

---
 modules/sd_hijack_optimizations.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py
index 79b8e3b47..e9d065a32 100644
--- a/modules/sd_hijack_optimizations.py
+++ b/modules/sd_hijack_optimizations.py
@@ -202,7 +202,7 @@ def einsum_op_cuda(q, k, v):
     return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20))
 
 def einsum_op_dml(q, k, v):
-    mem_total, mem_active = devices.adl.memory_stats()
+    mem_total, mem_active = torch.dml.memory_stats(q.device)
     mem_reserved = mem_total / (1 << 20) * 0.7
     return einsum_op_tensor_mem(q, k, v, (mem_reserved - mem_active) if mem_reserved > mem_active else 1)
 

From df0e89be480f67e63a27061c174de835e98f0910 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Wed, 26 Apr 2023 12:45:44 +0900
Subject: [PATCH 11/69] fix. Unstable & need more test.

---
 modules/dml/__init__.py                     | 12 +++++-------
 modules/dml/hijack/__init__.py              |  3 +++
 modules/dml/{ => hijack}/kdiffusion.py      |  0
 modules/dml/{ => hijack}/stablediffusion.py |  0
 modules/dml/{ => hijack}/torch.py           |  0
 modules/sd_hijack_optimizations.py          |  3 ---
 6 files changed, 8 insertions(+), 10 deletions(-)
 create mode 100644 modules/dml/hijack/__init__.py
 rename modules/dml/{ => hijack}/kdiffusion.py (100%)
 rename modules/dml/{ => hijack}/stablediffusion.py (100%)
 rename modules/dml/{ => hijack}/torch.py (100%)

diff --git a/modules/dml/__init__.py b/modules/dml/__init__.py
index fbda39384..90beb96d7 100644
--- a/modules/dml/__init__.py
+++ b/modules/dml/__init__.py
@@ -1,11 +1,9 @@
 import torch
 import torch_directml
 
-import modules.dml.kdiffusion
-import modules.dml.stablediffusion
-import modules.dml.torch
+import modules.dml.hijack
 
-from optimizer.unknown import UnknownOptimizer
+from modules.dml.optimizer.unknown import UnknownOptimizer
 
 class DirectML():
     def get_optimizer(self, device: torch.device):
@@ -13,11 +11,11 @@ class DirectML():
         try:
             device_name = torch_directml.device_name(device.index)
             if 'NVIDIA' in device_name or 'GeForce' in device_name:
-                from optimizer.nvidia import nVidiaOptimizer as optimizer
+                from modules.dml.optimizer.nvidia import nVidiaOptimizer as optimizer
             elif 'AMD' in device_name or 'Radeon' in device_name:
-                from optimizer.amd import AMDOptimizer as optimizer
+                from modules.dml.optimizer.amd import AMDOptimizer as optimizer
             elif 'Intel' in device_name:
-                from optimizer.intel import IntelOptimizer as optimizer
+                from modules.dml.optimizer.intel import IntelOptimizer as optimizer
             else:
                 return UnknownOptimizer
             return optimizer
diff --git a/modules/dml/hijack/__init__.py b/modules/dml/hijack/__init__.py
new file mode 100644
index 000000000..5cf59a704
--- /dev/null
+++ b/modules/dml/hijack/__init__.py
@@ -0,0 +1,3 @@
+import modules.dml.hijack.kdiffusion
+import modules.dml.hijack.stablediffusion
+import modules.dml.hijack.torch
diff --git a/modules/dml/kdiffusion.py b/modules/dml/hijack/kdiffusion.py
similarity index 100%
rename from modules/dml/kdiffusion.py
rename to modules/dml/hijack/kdiffusion.py
diff --git a/modules/dml/stablediffusion.py b/modules/dml/hijack/stablediffusion.py
similarity index 100%
rename from modules/dml/stablediffusion.py
rename to modules/dml/hijack/stablediffusion.py
diff --git a/modules/dml/torch.py b/modules/dml/hijack/torch.py
similarity index 100%
rename from modules/dml/torch.py
rename to modules/dml/hijack/torch.py
diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py
index e9d065a32..12ee9f956 100644
--- a/modules/sd_hijack_optimizations.py
+++ b/modules/sd_hijack_optimizations.py
@@ -20,9 +20,6 @@ if shared.opts.cross_attention_optimization == "xFormers":
     except Exception:
         pass
 
-if shared.device.type == 'privateuseone':
-    import dml
-
 
 def get_available_vram():
     if shared.device.type == 'cuda':

From d2d5011bd3cf7c982b92ef9019c5bb30d89f2bf2 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Wed, 26 Apr 2023 17:44:32 +0900
Subject: [PATCH 12/69] Implement memory estimation for AMDGPUs. Stable.

---
 modules/devices.py                            |  2 +-
 modules/dml/__init__.py                       | 14 ++--
 modules/dml/optimizer/amd/__init__.py         |  4 +-
 modules/dml/optimizer/amd/driver/atiadlxx.py  |  5 +-
 .../dml/optimizer/amd/driver/atiadlxx_apis.py | 67 ++++++++-----------
 modules/dml/optimizer/amd/memory.py           |  0
 modules/dml/optimizer/intel/__init__.py       |  2 +-
 modules/dml/optimizer/nvidia/__init__.py      |  2 +-
 modules/dml/optimizer/optimizer.py            |  2 +-
 modules/dml/optimizer/unknown/__init__.py     |  2 +-
 10 files changed, 47 insertions(+), 53 deletions(-)
 delete mode 100644 modules/dml/optimizer/amd/memory.py

diff --git a/modules/devices.py b/modules/devices.py
index 5f5e18768..e317d91f4 100644
--- a/modules/devices.py
+++ b/modules/devices.py
@@ -31,7 +31,7 @@ def get_dml_device_string():
     from modules import shared
     if shared.cmd_opts.device_id is not None:
         return f"privateuseone:{shared.cmd_opts.device_id}"
-    return "privateuseone"
+    return "privateuseone:0"
 
 
 def get_optimal_device_name():
diff --git a/modules/dml/__init__.py b/modules/dml/__init__.py
index 90beb96d7..b77db18d0 100644
--- a/modules/dml/__init__.py
+++ b/modules/dml/__init__.py
@@ -3,27 +3,27 @@ import torch_directml
 
 import modules.dml.hijack
 
-from modules.dml.optimizer.unknown import UnknownOptimizer
+from .optimizer.unknown import UnknownOptimizer
 
 class DirectML():
-    def get_optimizer(self, device: torch.device):
+    def get_optimizer(device: torch.device):
         assert(device.type == 'privateuseone')
         try:
             device_name = torch_directml.device_name(device.index)
             if 'NVIDIA' in device_name or 'GeForce' in device_name:
-                from modules.dml.optimizer.nvidia import nVidiaOptimizer as optimizer
+                from .optimizer.nvidia import nVidiaOptimizer as optimizer
             elif 'AMD' in device_name or 'Radeon' in device_name:
-                from modules.dml.optimizer.amd import AMDOptimizer as optimizer
+                from .optimizer.amd import AMDOptimizer as optimizer
             elif 'Intel' in device_name:
-                from modules.dml.optimizer.intel import IntelOptimizer as optimizer
+                from .optimizer.intel import IntelOptimizer as optimizer
             else:
                 return UnknownOptimizer
             return optimizer
         except:
             return UnknownOptimizer
 
-    def memory_stats(self, device: torch.device):
-        optimizer = self.get_optimizer(device)
+    def memory_stats(device: torch.device):
+        optimizer = DirectML.get_optimizer(device)
         return optimizer.memory_stats(device.index)
 
 # Alternative of torch.cuda for DirectML.
diff --git a/modules/dml/optimizer/amd/__init__.py b/modules/dml/optimizer/amd/__init__.py
index 37ea3ea91..23c6e57e3 100644
--- a/modules/dml/optimizer/amd/__init__.py
+++ b/modules/dml/optimizer/amd/__init__.py
@@ -1,7 +1,7 @@
 from modules.dml.optimizer.optimizer import Optimizer
-from driver.atiadlxx import ATIADLxx
+from .driver.atiadlxx import ATIADLxx
 
 class AMDOptimizer(Optimizer):
     driver: ATIADLxx = ATIADLxx()
-    def memory_stats(self, index):
+    def memory_stats(index):
         return (AMDOptimizer.driver.iHyperMemorySize, AMDOptimizer.driver.get_dedicated_vram_usage(index))
diff --git a/modules/dml/optimizer/amd/driver/atiadlxx.py b/modules/dml/optimizer/amd/driver/atiadlxx.py
index abb5eca96..81a578a5c 100644
--- a/modules/dml/optimizer/amd/driver/atiadlxx.py
+++ b/modules/dml/optimizer/amd/driver/atiadlxx.py
@@ -14,8 +14,11 @@ class ATIADLxx(object):
         AdapterInfoArray = (AdapterInfo * num_adapters.value)()
         ADL2_Adapter_AdapterInfo_Get(self.context, C.cast(AdapterInfoArray, LPAdapterInfo), C.sizeof(AdapterInfoArray))
         self.devices = []
+        busNumbers = []
         for adapter in AdapterInfoArray:
-            self.devices.append(adapter)
+            if adapter.iBusNumber not in busNumbers: # filter duplicate device
+                self.devices.append(adapter)
+                busNumbers.append(adapter.iBusNumber)
         self.iHyperMemorySize = self.get_memory_info2(0).iHyperMemorySize
 
     def get_memory_info2(self, adapterIndex: int) -> ADLMemoryInfo2:
diff --git a/modules/dml/optimizer/amd/driver/atiadlxx_apis.py b/modules/dml/optimizer/amd/driver/atiadlxx_apis.py
index 23b7da3b1..fef70b543 100644
--- a/modules/dml/optimizer/amd/driver/atiadlxx_apis.py
+++ b/modules/dml/optimizer/amd/driver/atiadlxx_apis.py
@@ -1,50 +1,41 @@
 import ctypes as C
-import platform
 from .atiadlxx_structures import *
 
-_platform = platform.system()
+atiadlxx = C.WinDLL("atiadlxx.dll")
 
-try:
-    if _platform == "Windows":
-        atiadlxx = C.WinDLL("atiadlxx.dll")
+ADL_MAIN_MALLOC_CALLBACK = C.CFUNCTYPE(C.c_void_p, C.c_int)
+ADL_MAIN_FREE_CALLBACK = C.CFUNCTYPE(None, C.POINTER(C.c_void_p))
 
-        ADL_MAIN_MALLOC_CALLBACK = C.CFUNCTYPE(C.c_void_p, C.c_int)
-        ADL_MAIN_FREE_CALLBACK = C.CFUNCTYPE(None, C.POINTER(C.c_void_p))
+@ADL_MAIN_MALLOC_CALLBACK
+def ADL_Main_Memory_Alloc(iSize):
+    return C._malloc(iSize)
 
-        @ADL_MAIN_MALLOC_CALLBACK
-        def ADL_Main_Memory_Alloc(iSize):
-            return C._malloc(iSize)
+@ADL_MAIN_FREE_CALLBACK
+def ADL_Main_Memory_Free(lpBuffer):
+    if lpBuffer[0] is not None:
+        C._free(lpBuffer[0])
+        lpBuffer[0] = None
 
-        @ADL_MAIN_FREE_CALLBACK
-        def ADL_Main_Memory_Free(lpBuffer):
-            if lpBuffer[0] is not None:
-                C._free(lpBuffer[0])
-                lpBuffer[0] = None
+ADL2_Main_Control_Create = atiadlxx.ADL2_Main_Control_Create
+ADL2_Main_Control_Create.restype = C.c_int
+ADL2_Main_Control_Create.argtypes = [ADL_MAIN_MALLOC_CALLBACK, C.c_int, ADL_CONTEXT_HANDLE]
 
-        ADL2_Main_Control_Create = atiadlxx.ADL2_Main_Control_Create
-        ADL2_Main_Control_Create.restype = C.c_int
-        ADL2_Main_Control_Create.argtypes = [ADL_MAIN_MALLOC_CALLBACK, C.c_int, ADL_CONTEXT_HANDLE]
+ADL2_Adapter_NumberOfAdapters_Get = atiadlxx.ADL2_Adapter_NumberOfAdapters_Get
+ADL2_Adapter_NumberOfAdapters_Get.restype = C.c_int
+ADL2_Adapter_NumberOfAdapters_Get.argtypes = [ADL_CONTEXT_HANDLE, C.POINTER(C.c_int)]
 
-        ADL2_Adapter_NumberOfAdapters_Get = atiadlxx.ADL2_Adapter_NumberOfAdapters_Get
-        ADL2_Adapter_NumberOfAdapters_Get.restype = C.c_int
-        ADL2_Adapter_NumberOfAdapters_Get.argtypes = [ADL_CONTEXT_HANDLE, C.POINTER(C.c_int)]
+ADL2_Adapter_AdapterInfo_Get = atiadlxx.ADL2_Adapter_AdapterInfo_Get
+ADL2_Adapter_AdapterInfo_Get.restype = C.c_int
+ADL2_Adapter_AdapterInfo_Get.argtypes = [ADL_CONTEXT_HANDLE, LPAdapterInfo, C.c_int]
 
-        ADL2_Adapter_AdapterInfo_Get = atiadlxx.ADL2_Adapter_AdapterInfo_Get
-        ADL2_Adapter_AdapterInfo_Get.restype = C.c_int
-        ADL2_Adapter_AdapterInfo_Get.argtypes = [ADL_CONTEXT_HANDLE, LPAdapterInfo, C.c_int]
+ADL2_Adapter_MemoryInfo2_Get = atiadlxx.ADL2_Adapter_MemoryInfo2_Get
+ADL2_Adapter_MemoryInfo2_Get.restype = C.c_int
+ADL2_Adapter_MemoryInfo2_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(ADLMemoryInfo2)]
 
-        ADL2_Adapter_MemoryInfo2_Get = atiadlxx.ADL2_Adapter_MemoryInfo2_Get
-        ADL2_Adapter_MemoryInfo2_Get.restype = C.c_int
-        ADL2_Adapter_MemoryInfo2_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(ADLMemoryInfo2)]
+ADL2_Adapter_DedicatedVRAMUsage_Get = atiadlxx.ADL2_Adapter_DedicatedVRAMUsage_Get
+ADL2_Adapter_DedicatedVRAMUsage_Get.restype = C.c_int
+ADL2_Adapter_DedicatedVRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)]
 
-        ADL2_Adapter_DedicatedVRAMUsage_Get = atiadlxx.ADL2_Adapter_DedicatedVRAMUsage_Get
-        ADL2_Adapter_DedicatedVRAMUsage_Get.restype = C.c_int
-        ADL2_Adapter_DedicatedVRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)]
-
-        ADL2_Adapter_VRAMUsage_Get = atiadlxx.ADL2_Adapter_VRAMUsage_Get
-        ADL2_Adapter_VRAMUsage_Get.restype = C.c_int
-        ADL2_Adapter_VRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)]
-    else:
-        print("Warning: experimental graphic memory optimization for AMDGPU is disabled. Because this is not Windows platform.")
-except FileNotFoundError:
-    print("Warning: memory optimization for AMDGPU is disabled. Because couldn't find 'atiadlxx.dll'. Please install GPU driver downloaded from AMD.com.")
\ No newline at end of file
+ADL2_Adapter_VRAMUsage_Get = atiadlxx.ADL2_Adapter_VRAMUsage_Get
+ADL2_Adapter_VRAMUsage_Get.restype = C.c_int
+ADL2_Adapter_VRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)]
diff --git a/modules/dml/optimizer/amd/memory.py b/modules/dml/optimizer/amd/memory.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/modules/dml/optimizer/intel/__init__.py b/modules/dml/optimizer/intel/__init__.py
index d17cd59d2..bffda69f1 100644
--- a/modules/dml/optimizer/intel/__init__.py
+++ b/modules/dml/optimizer/intel/__init__.py
@@ -1,7 +1,7 @@
 from modules.dml.optimizer.optimizer import Optimizer
 
 class IntelOptimizer(Optimizer):
-    def memory_stats():
+    def memory_stats(index):
         raise NotImplementedError()
         # DML TODO: Implement
         return
diff --git a/modules/dml/optimizer/nvidia/__init__.py b/modules/dml/optimizer/nvidia/__init__.py
index d467e8ae5..e5fda97b8 100644
--- a/modules/dml/optimizer/nvidia/__init__.py
+++ b/modules/dml/optimizer/nvidia/__init__.py
@@ -1,7 +1,7 @@
 from modules.dml.optimizer.optimizer import Optimizer
 
 class nVidiaOptimizer(Optimizer):
-    def memory_stats():
+    def memory_stats(index):
         raise NotImplementedError()
         # DML TODO: Implement
         return
diff --git a/modules/dml/optimizer/optimizer.py b/modules/dml/optimizer/optimizer.py
index 2b2d9ed64..db88c6579 100644
--- a/modules/dml/optimizer/optimizer.py
+++ b/modules/dml/optimizer/optimizer.py
@@ -4,5 +4,5 @@ from typing import *
 class Optimizer(metaclass=ABCMeta):
     driver: Any = None
     @abstractmethod
-    def memory_stats(self, index: int) -> Tuple[int, int]:
+    def memory_stats(index: int) -> Tuple[int, int]:
         pass
diff --git a/modules/dml/optimizer/unknown/__init__.py b/modules/dml/optimizer/unknown/__init__.py
index f2fbc2ae2..40ff476cd 100644
--- a/modules/dml/optimizer/unknown/__init__.py
+++ b/modules/dml/optimizer/unknown/__init__.py
@@ -1,6 +1,6 @@
 from modules.dml.optimizer.optimizer import Optimizer
 
 class UnknownOptimizer(Optimizer):
-    def memory_stats():
+    def memory_stats(index):
         # DML TODO: Implement
         return (1073741824, 0)

From c9dde03fc54d0882679b90c6e717cf11b5969433 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Wed, 26 Apr 2023 18:54:24 +0900
Subject: [PATCH 13/69] Move realesrgan fallback to dml/hijack.

---
 TODO_DML.md                            | 33 ------------
 modules/dml/hijack/__init__.py         |  1 +
 modules/dml/hijack/realesrgan_model.py | 69 ++++++++++++++++++++++++++
 modules/realesrgan_model.py            | 68 -------------------------
 setup.py                               |  6 +--
 5 files changed, 73 insertions(+), 104 deletions(-)
 delete mode 100644 TODO_DML.md
 create mode 100644 modules/dml/hijack/realesrgan_model.py

diff --git a/TODO_DML.md b/TODO_DML.md
deleted file mode 100644
index f359ba1b3..000000000
--- a/TODO_DML.md
+++ /dev/null
@@ -1,33 +0,0 @@
-# TODO
-
-## Issues
-
-Stuff to be fixed...
-
-- `mat1 and mat2 must have the same dtype` error (half mode)
-- Some samplers won't work (test later)
-
-## Something needs discussion
-
-- About memory optimization.
-
-Basically, we cannot get detailed vram information from `torch-directml`.
-
-It has `gpu_memory` method which returns an array contains used memory size, but it is almostly useless without any other information.
-
-What should we do?
-
-1. Use any fixed value as the available memory capacity.
-2. Use `atiadlxx`(AMD/ATI GPU driver library) to infer vram information as similar as possible to the actual value. (works for AMDGPUs)
-3. or another better way.
-
-- Half mode does not work well.
-
-In half precision, we get an error like `mat1 and mat2 must have the same dtype`.
-
-I solved this problem by overriding forward of GroupNorm / LayerNorm / Conv2d / Linear to convert input tensor to full precision (and convert to float16 before return).
-
-What should we do?
-
-1. Override forwards (same solution)
-2. When using DirectML, set the default to full precision and notify the user.
diff --git a/modules/dml/hijack/__init__.py b/modules/dml/hijack/__init__.py
index 5cf59a704..dd71784c0 100644
--- a/modules/dml/hijack/__init__.py
+++ b/modules/dml/hijack/__init__.py
@@ -1,3 +1,4 @@
 import modules.dml.hijack.kdiffusion
 import modules.dml.hijack.stablediffusion
 import modules.dml.hijack.torch
+import modules.dml.hijack.realesrgan_model
diff --git a/modules/dml/hijack/realesrgan_model.py b/modules/dml/hijack/realesrgan_model.py
new file mode 100644
index 000000000..c82429fda
--- /dev/null
+++ b/modules/dml/hijack/realesrgan_model.py
@@ -0,0 +1,69 @@
+import math
+import torch
+
+from realesrgan import RealESRGANer
+
+
+# DML Solution: Some tensors turn 0 after Extended Slices. Move output to cpu and get it back.
+def tile_process(self):
+    batch, channel, height, width = self.img.shape
+    output_height = height * self.scale
+    output_width = width * self.scale
+    output_shape = (batch, channel, output_height, output_width)
+
+    # start with black image
+    self.output = self.img.new_zeros(output_shape, device='cpu')
+    tiles_x = math.ceil(width / self.tile_size)
+    tiles_y = math.ceil(height / self.tile_size)
+
+    # loop over all tiles
+    for y in range(tiles_y):
+        for x in range(tiles_x):
+            # extract tile from input image
+            ofs_x = x * self.tile_size
+            ofs_y = y * self.tile_size
+            # input tile area on total image
+            input_start_x = ofs_x
+            input_end_x = min(ofs_x + self.tile_size, width)
+            input_start_y = ofs_y
+            input_end_y = min(ofs_y + self.tile_size, height)
+
+            # input tile area on total image with padding
+            input_start_x_pad = max(input_start_x - self.tile_pad, 0)
+            input_end_x_pad = min(input_end_x + self.tile_pad, width)
+            input_start_y_pad = max(input_start_y - self.tile_pad, 0)
+            input_end_y_pad = min(input_end_y + self.tile_pad, height)
+
+            # input tile dimensions
+            input_tile_width = input_end_x - input_start_x
+            input_tile_height = input_end_y - input_start_y
+            tile_idx = y * tiles_x + x + 1
+            input_tile = self.img[:, :, input_start_y_pad:input_end_y_pad, input_start_x_pad:input_end_x_pad]
+
+            # upscale tile
+            try:
+                with torch.no_grad():
+                    output_tile = self.model(input_tile)
+                    output_tile = output_tile.cpu()
+            except RuntimeError as error:
+                print('Error', error)
+            print(f'\tTile {tile_idx}/{tiles_x * tiles_y}')
+
+            # output tile area on total image
+            output_start_x = input_start_x * self.scale
+            output_end_x = input_end_x * self.scale
+            output_start_y = input_start_y * self.scale
+            output_end_y = input_end_y * self.scale
+
+            # output tile area without padding
+            output_start_x_tile = (input_start_x - input_start_x_pad) * self.scale
+            output_end_x_tile = output_start_x_tile + input_tile_width * self.scale
+            output_start_y_tile = (input_start_y - input_start_y_pad) * self.scale
+            output_end_y_tile = output_start_y_tile + input_tile_height * self.scale
+
+            # put tile into output image
+            self.output[:, :, output_start_y:output_end_y,
+                        output_start_x:output_end_x] = output_tile[:, :, output_start_y_tile:output_end_y_tile,
+                                                                    output_start_x_tile:output_end_x_tile]
+    self.output = self.output.to(self.device)
+RealESRGANer.tile_process = tile_process
diff --git a/modules/realesrgan_model.py b/modules/realesrgan_model.py
index 1d48ac42c..398f2526b 100644
--- a/modules/realesrgan_model.py
+++ b/modules/realesrgan_model.py
@@ -10,72 +10,6 @@ from modules.shared import cmd_opts, opts, device
 import modules.errors as errors
 
 
-# DML Solution: Some tensors turn 0 after Extended Slices. Move output to cpu and get it back.
-def realesrgan_tile_process_dml_fix(self):
-    import math
-    import torch
-    batch, channel, height, width = self.img.shape
-    output_height = height * self.scale
-    output_width = width * self.scale
-    output_shape = (batch, channel, output_height, output_width)
-
-    # start with black image
-    self.output = self.img.new_zeros(output_shape, device='cpu' if self.device.type == 'privateuseone' else self.device)
-    tiles_x = math.ceil(width / self.tile_size)
-    tiles_y = math.ceil(height / self.tile_size)
-
-    # loop over all tiles
-    for y in range(tiles_y):
-        for x in range(tiles_x):
-            # extract tile from input image
-            ofs_x = x * self.tile_size
-            ofs_y = y * self.tile_size
-            # input tile area on total image
-            input_start_x = ofs_x
-            input_end_x = min(ofs_x + self.tile_size, width)
-            input_start_y = ofs_y
-            input_end_y = min(ofs_y + self.tile_size, height)
-
-            # input tile area on total image with padding
-            input_start_x_pad = max(input_start_x - self.tile_pad, 0)
-            input_end_x_pad = min(input_end_x + self.tile_pad, width)
-            input_start_y_pad = max(input_start_y - self.tile_pad, 0)
-            input_end_y_pad = min(input_end_y + self.tile_pad, height)
-
-            # input tile dimensions
-            input_tile_width = input_end_x - input_start_x
-            input_tile_height = input_end_y - input_start_y
-            tile_idx = y * tiles_x + x + 1
-            input_tile = self.img[:, :, input_start_y_pad:input_end_y_pad, input_start_x_pad:input_end_x_pad]
-
-            # upscale tile
-            try:
-                with torch.no_grad():
-                    output_tile = self.model(input_tile)
-                    output_tile = output_tile.cpu()
-            except RuntimeError as error:
-                print('Error', error)
-            print(f'\tTile {tile_idx}/{tiles_x * tiles_y}')
-
-            # output tile area on total image
-            output_start_x = input_start_x * self.scale
-            output_end_x = input_end_x * self.scale
-            output_start_y = input_start_y * self.scale
-            output_end_y = input_end_y * self.scale
-
-            # output tile area without padding
-            output_start_x_tile = (input_start_x - input_start_x_pad) * self.scale
-            output_end_x_tile = output_start_x_tile + input_tile_width * self.scale
-            output_start_y_tile = (input_start_y - input_start_y_pad) * self.scale
-            output_end_y_tile = output_start_y_tile + input_tile_height * self.scale
-
-            # put tile into output image
-            self.output[:, :, output_start_y:output_end_y,
-                        output_start_x:output_end_x] = output_tile[:, :, output_start_y_tile:output_end_y_tile,
-                                                                    output_start_x_tile:output_end_x_tile]
-    self.output = self.output.to(self.device)
-
-
 class UpscalerRealESRGAN(Upscaler):
     def __init__(self, path):
         self.name = "RealESRGAN"
@@ -103,8 +37,6 @@ class UpscalerRealESRGAN(Upscaler):
 
         try:
             from realesrgan import RealESRGANer
-            if device.type == 'privateuseone':
-                RealESRGANer.tile_process = realesrgan_tile_process_dml_fix
         except:
             print("Error importing Real-ESRGAN:", file=sys.stderr)
             return img
diff --git a/setup.py b/setup.py
index 69b86012d..52da5d8e3 100644
--- a/setup.py
+++ b/setup.py
@@ -266,12 +266,12 @@ def install_repositories():
     stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git")
     stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf")
     clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit)
-    k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git')
-    k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919")
-    clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit)
     taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git")
     taming_transformers_commit = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "3ba01b241669f5ade541ce990f7650a3b8f65318")
     clone(taming_transformers_repo, d('taming-transformers'), taming_transformers_commit)
+    k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git')
+    k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919")
+    clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit)
     codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git')
     codeformer_commit = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af")
     clone(codeformer_repo, d('CodeFormer'), codeformer_commit)

From 576ea88618d4f752e83483b2c30f541c6ba8747d Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Fri, 28 Apr 2023 10:56:00 +0900
Subject: [PATCH 14/69] torch 2.0.0 support.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 99d079463..dda7a4423 100644
--- a/setup.py
+++ b/setup.py
@@ -200,7 +200,7 @@ def check_torch():
         machine = platform.machine()
         if 'arm' not in machine and 'aarch' not in machine and not args.nodirectml: # torch-directml is available on AMD64
             log.info('Using DirectML Backend')
-            torch_command = os.environ.get('TORCH_COMMAND', 'torch==1.13.1 torchvision==0.14.1 torch-directml')
+            torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.0 torchvision torch-directml')
             xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
         else:
             log.info('Using CPU-only Torch')

From 9a09b2eef30264490a0b6f155f8044a47cfe2a04 Mon Sep 17 00:00:00 2001
From: cool-bigdogs-tshirt
 <131823432+cool-bigdogs-tshirt@users.noreply.github.com>
Date: Thu, 27 Apr 2023 11:24:26 -0500
Subject: [PATCH 15/69] attempt at unipc latent upscaling

i should have taken linear algebra before i dropped out...
---
 modules/models/diffusion/uni_pc/sampler.py | 74 ++++++++++++++++++++++
 modules/processing.py                      |  3 +-
 2 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/modules/models/diffusion/uni_pc/sampler.py b/modules/models/diffusion/uni_pc/sampler.py
index a241c8a7c..41b8c9a5b 100644
--- a/modules/models/diffusion/uni_pc/sampler.py
+++ b/modules/models/diffusion/uni_pc/sampler.py
@@ -4,6 +4,7 @@ import torch
 
 from .uni_pc import NoiseScheduleVP, model_wrapper, UniPC
 from modules import shared, devices
+from ldm.modules.diffusionmodules.util import extract_into_tensor
 
 
 class UniPCSampler(object):
@@ -15,6 +16,79 @@ class UniPCSampler(object):
         self.after_sample = None
         self.register_buffer('alphas_cumprod', to_torch(model.alphas_cumprod))
 
+    def make_schedule(self, ddim_num_steps, ddim_discretize="uniform", ddim_eta=0., verbose=True):
+        # persist steps so we can eventually find denoising strength
+        self.inflated_steps = ddim_num_steps
+
+    @torch.no_grad()
+    def stochastic_encode(self, x0, t, use_original_steps=False, noise=None):
+        if noise is None:
+            noise = torch.randn_like(x0)
+
+        # first time we have all the info to get the real parameters from the ui
+        hires_steps = t[0] + 1
+        inflated_steps = self.inflated_steps
+        self.denoising_strength = hires_steps/inflated_steps
+
+        adjusted_steps = int(hires_steps * self.denoising_strength)
+        self.steps = max(adjusted_steps, shared.opts.uni_pc_order+1)
+
+        t = torch.full(t.shape, self.steps).to(t.device)
+
+        timesteps = torch.asarray(list(range(
+            t,
+            self.model.num_timesteps,
+            self.model.num_timesteps // hires_steps,
+        ))) + 1
+        alphas = self.model.alphas_cumprod[timesteps]
+        sqrt_one_minus_alphas = torch.sqrt(1. - alphas)
+        a = extract_into_tensor(torch.sqrt(alphas), t, x0.shape) * x0 
+        b = extract_into_tensor(sqrt_one_minus_alphas, t, x0.shape) * noise
+
+        return (a+b)
+
+    def decode(self, x_latent, conditioning, t_start, unconditional_guidance_scale=1.0, unconditional_conditioning=None,
+               use_original_steps=False, callback=None):
+        #print(f'steps {self.steps} denoising {self.denoising_strength}')
+
+        noise_schedule = NoiseScheduleVP("discrete", alphas_cumprod=self.alphas_cumprod)
+
+        # same as in .sample(), i guess
+        model_type = "v" if self.model.parameterization == "v" else "noise"
+
+        model_fn = model_wrapper(
+            lambda x, t, c: self.model.apply_model(x, t, c),
+            noise_schedule,
+            model_type=model_type,
+            guidance_type="classifier-free",
+            #condition=conditioning,
+            #unconditional_condition=unconditional_conditioning,
+            guidance_scale=unconditional_guidance_scale,
+        )
+
+        self.uni_pc = UniPC(
+                model_fn,
+                noise_schedule,
+                predict_x0=True,
+                thresholding=False,
+                variant=shared.opts.uni_pc_variant,
+                condition=conditioning,
+                unconditional_condition=unconditional_conditioning,
+                before_sample=self.before_sample,
+                after_sample=self.after_sample,
+                after_update=self.after_update,
+            )
+
+        return self.uni_pc.sample(
+                x_latent,
+                steps=self.steps,
+                skip_type=shared.opts.uni_pc_skip_type,
+                method="multistep",
+                order=shared.opts.uni_pc_order,
+                lower_order_final=shared.opts.uni_pc_lower_order_final,
+                t_start=self.denoising_strength,
+            )
+
     def register_buffer(self, name, attr):
         if type(attr) == torch.Tensor:
             if attr.device != devices.device:
diff --git a/modules/processing.py b/modules/processing.py
index 04379fabe..c3ea4b2b8 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -970,7 +970,8 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
         shared.state.nextjob()
 
         img2img_sampler_name = self.sampler_name
-        if self.sampler_name in ['PLMS', 'UniPC']:  # PLMS/UniPC do not support img2img so we just silently switch to DDIM
+        if self.sampler_name in ['PLMS']:
+            # PLMS does not support img2img, use fallback instead
             img2img_sampler_name = shared.opts.fallback_sampler
         self.sampler = sd_samplers.create_sampler(img2img_sampler_name, self.sd_model)
 

From d3f0294bde458778d51010bf990848ee8a993881 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Fri, 28 Apr 2023 22:01:29 -0400
Subject: [PATCH 16/69] update options

---
 javascript/black-orange.css |  1 +
 javascript/notification.js  | 28 ++++------------------------
 modules/cmd_args.py         |  1 -
 modules/shared.py           |  8 ++++----
 4 files changed, 9 insertions(+), 29 deletions(-)

diff --git a/javascript/black-orange.css b/javascript/black-orange.css
index 52e601529..a055e2e51 100644
--- a/javascript/black-orange.css
+++ b/javascript/black-orange.css
@@ -103,6 +103,7 @@ svg.feather.feather-image, .feather .feather-image { display: none }
 #txt2img_tools, #img2img_tools { margin-top: 54px; scale: 120%; margin-left: 26px; }
 #txtimg_hr_finalres { max-width: 200px; }
 #pnginfo_html2_info { margin-top: -18px; background-color: var(--input-background-fill); padding: var(--input-padding) }
+#txt2img_extra_refresh, #txt2img_extra_close { height: 1.7em }
 
 /* custom elements overrides */
 #steps-animation, #controlnet { border-width: 0; }
diff --git a/javascript/notification.js b/javascript/notification.js
index 3b68ffe53..9f7c2e439 100644
--- a/javascript/notification.js
+++ b/javascript/notification.js
@@ -1,52 +1,32 @@
 // Monitors the gallery and sends a browser notification when the leading image is new.
 
 let lastHeadImg = null;
-
 let notificationButton = null;
-
 const regExpTempImage = /(?<=\/|\\)tmp[\w\d]{8}\.png$/gm;
 
 onUiUpdate(function(){
     if(notificationButton == null){
         notificationButton = gradioApp().getElementById('request_notifications')
-
-        if(notificationButton != null){
-            notificationButton.addEventListener('click', function (evt) {
-                Notification.requestPermission();
-            },true);
-        }
+        if (notificationButton != null) notificationButton.addEventListener('click', (evt) => Notification.requestPermission(), true);
     }
-
     const galleryPreviews = gradioApp().querySelectorAll('div[id^="tab_"][style*="display: block"] div[id$="_results"] .thumbnail-item > img');
-
     if (galleryPreviews == null) return;
-
     const headImg = galleryPreviews[0]?.src;
-
     if (headImg == null || headImg == lastHeadImg) return;
-
     if (headImg.search(regExpTempImage) != -1) return;
-
     lastHeadImg = headImg;
-
     // play notification sound if available
     gradioApp().querySelector('#audio_notification audio')?.play();
-
     if (document.hasFocus()) return;
-
     // Multiple copies of the images are in the DOM when one is selected. Dedup with a Set to get the real number generated.
     const imgs = new Set(Array.from(galleryPreviews).map(img => img.src));
-
     const notification = new Notification(
-        'Stable Diffusion',
-        {
+        'Stable Diffusion', {
             body: `Generated ${imgs.size > 1 ? imgs.size - opts.return_grid : 1} image${imgs.size > 1 ? 's' : ''}`,
             icon: headImg,
-            image: headImg,
-        }
+            image: headImg }
     );
-
-    notification.onclick = function(_){
+    notification.onclick = function(_) {
         parent.focus();
         this.close();
     };
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index 87b47d6c7..65d4d4cfe 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -78,7 +78,6 @@ def compatibility_args(opts, args):
     opts.use_old_emphasis_implementation = False
     opts.use_old_karras_scheduler_sigmas = False
     opts.no_dpmpp_sde_batch_determinism = False
-    opts.use_old_hires_fix_width_height = False
 
     parser.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir)
     args = parser.parse_args()
diff --git a/modules/shared.py b/modules/shared.py
index d3720db6b..5c94bc4ff 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -330,11 +330,11 @@ options_templates.update(options_section(('cuda', "CUDA Settings"), {
 }))
 
 options_templates.update(options_section(('upscaling', "Upscaling"), {
-    "ESRGAN_tile": OptionInfo(192, "Tile size for ESRGAN upscalers. 0 = no tiling.", gr.Slider, {"minimum": 0, "maximum": 512, "step": 16}),
-    "ESRGAN_tile_overlap": OptionInfo(8, "Tile overlap, in pixels for ESRGAN upscalers. Low values = visible seam.", gr.Slider, {"minimum": 0, "maximum": 48, "step": 1}),
-    "realesrgan_enabled_models": OptionInfo(["R-ESRGAN 4x+", "R-ESRGAN 4x+ Anime6B"], "Select which Real-ESRGAN models to show in the web UI.", gr.CheckboxGroup, lambda: {"choices": shared_items.realesrgan_models_names()}),
+    "ESRGAN_tile": OptionInfo(192, "Tile size for ESRGAN upscalers (0 = no tiling)", gr.Slider, {"minimum": 0, "maximum": 512, "step": 16}),
+    "ESRGAN_tile_overlap": OptionInfo(8, "Tile overlap in pixels for ESRGAN upscalers", gr.Slider, {"minimum": 0, "maximum": 48, "step": 1}),
+    "realesrgan_enabled_models": OptionInfo(["R-ESRGAN 4x+", "R-ESRGAN 4x+ Anime6B"], "Real-ESRGAN available models", gr.CheckboxGroup, lambda: {"choices": shared_items.realesrgan_models_names()}),
     "upscaler_for_img2img": OptionInfo("None", "Default upscaler for image resize operations", gr.Dropdown, lambda: {"choices": [x.name for x in sd_upscalers]}),
-    "use_old_hires_fix_width_height": OptionInfo(False, "For hires fix, use width/height sliders to set final resolution rather than first pass (disables Upscale by, Resize width/height to)."),
+    "use_old_hires_fix_width_height": OptionInfo(False, "Hires fix uses width & height to set final resolution rather than first pass"),
     "dont_fix_second_order_samplers_schedule": OptionInfo(False, "Do not fix prompt schedule for second order samplers."),
 }))
 

From 42e30bfc3cf6b10664fbd66cabcd3c65f69e9cd4 Mon Sep 17 00:00:00 2001
From: cool-bigdogs-tshirt
 <131823432+cool-bigdogs-tshirt@users.noreply.github.com>
Date: Fri, 28 Apr 2023 22:31:53 -0500
Subject: [PATCH 17/69] unipc img2img - add a bunch of code to get a single
 value that maybe performs slightly better?

---
 modules/models/diffusion/uni_pc/sampler.py | 51 ++++++++++++++++------
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/modules/models/diffusion/uni_pc/sampler.py b/modules/models/diffusion/uni_pc/sampler.py
index 41b8c9a5b..3b468cf3f 100644
--- a/modules/models/diffusion/uni_pc/sampler.py
+++ b/modules/models/diffusion/uni_pc/sampler.py
@@ -1,5 +1,6 @@
 """SAMPLING ONLY."""
 
+import numpy as np
 import torch
 
 from .uni_pc import NoiseScheduleVP, model_wrapper, UniPC
@@ -26,26 +27,48 @@ class UniPCSampler(object):
             noise = torch.randn_like(x0)
 
         # first time we have all the info to get the real parameters from the ui
-        hires_steps = t[0] + 1
+        # value from the hires steps slider:
+        num_inference_steps = t[0] + 1
+        # (num_inference_steps // denoising_strength):
         inflated_steps = self.inflated_steps
-        self.denoising_strength = hires_steps/inflated_steps
+        # not exact:
+        self.denoising_strength = num_inference_steps/inflated_steps
 
-        adjusted_steps = int(hires_steps * self.denoising_strength)
-        self.steps = max(adjusted_steps, shared.opts.uni_pc_order+1)
+        # values used for timesteps that generate noise in diffusers repo
+        init_timestep = min(
+                int(num_inference_steps * self.denoising_strength),
+                num_inference_steps,
+            )
+        t_start = max(num_inference_steps - init_timestep, 0)
+
+        # actual number of steps we'll run
+        self.steps = max(
+                num_inference_steps - init_timestep,
+                shared.opts.uni_pc_order+1,
+            )
 
         t = torch.full(t.shape, self.steps).to(t.device)
 
-        timesteps = torch.asarray(list(range(
-            t,
-            self.model.num_timesteps,
-            self.model.num_timesteps // hires_steps,
-        ))) + 1
-        alphas = self.model.alphas_cumprod[timesteps]
-        sqrt_one_minus_alphas = torch.sqrt(1. - alphas)
-        a = extract_into_tensor(torch.sqrt(alphas), t, x0.shape) * x0 
-        b = extract_into_tensor(sqrt_one_minus_alphas, t, x0.shape) * noise
+        scheduler_timesteps = np.linspace(
+                0,
+                self.model.num_timesteps-1,
+                num_inference_steps + 1,
+            ).round()[::-1][:-1].copy().astype(np.int64)
+        _, unique_indices = np.unique(scheduler_timesteps, return_index=True)
+        scheduler_timesteps = scheduler_timesteps[np.sort(unique_indices)]
+        scheduler_timesteps = torch.from_numpy(scheduler_timesteps).to(t.device)
 
-        return (a+b)
+        sample_timesteps = scheduler_timesteps[t_start:]
+        latent_timestep = sample_timesteps[:1].repeat(x0.shape[0])
+
+        alphas_cumprod = self.alphas_cumprod
+        sqrt_alphas_prod = alphas_cumprod[latent_timestep] ** 0.5
+        sqrt_alphas_prod = sqrt_alphas_prod.flatten()
+
+        sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[latent_timestep]) ** 0.5
+        sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten()
+
+        return (sqrt_alphas_prod * x0 + sqrt_one_minus_alpha_prod * noise)
 
     def decode(self, x_latent, conditioning, t_start, unconditional_guidance_scale=1.0, unconditional_conditioning=None,
                use_original_steps=False, callback=None):

From a78ce0a3ca4521a5661042c47e3dd56eee7a1eeb Mon Sep 17 00:00:00 2001
From: cool-bigdogs-tshirt
 <131823432+cool-bigdogs-tshirt@users.noreply.github.com>
Date: Thu, 27 Apr 2023 12:04:14 -0500
Subject: [PATCH 18/69] xyz override for latent upscaler fallback

---
 modules/processing.py | 5 +++--
 scripts/xyz_grid.py   | 9 +++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/modules/processing.py b/modules/processing.py
index c3ea4b2b8..a83d9fa12 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -970,9 +970,10 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
         shared.state.nextjob()
 
         img2img_sampler_name = self.sampler_name
-        if self.sampler_name in ['PLMS']:
+        force_latent_upscaler = shared.opts.data.get('xyz_fallback_sampler')
+        if self.sampler_name in ['PLMS'] or force_latent_upscaler is not None:
             # PLMS does not support img2img, use fallback instead
-            img2img_sampler_name = shared.opts.fallback_sampler
+            img2img_sampler_name = force_latent_upscaler or shared.opts.fallback_sampler
         self.sampler = sd_samplers.create_sampler(img2img_sampler_name, self.sd_model)
 
         samples = samples[:, :, self.truncate_y//2:samples.shape[2]-(self.truncate_y+1)//2, self.truncate_x//2:samples.shape[3]-(self.truncate_x+1)//2]
diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py
index 52ae1c6e1..9a5a67241 100644
--- a/scripts/xyz_grid.py
+++ b/scripts/xyz_grid.py
@@ -128,6 +128,14 @@ def apply_styles(p: StableDiffusionProcessingTxt2Img, x: str, _):
     p.styles.extend(x.split(','))
 
 
+def apply_fallback(p, x, xs):
+    sampler_name = sd_samplers.samplers_map.get(x.lower(), None)
+    if sampler_name is None:
+        raise RuntimeError(f"Unknown sampler: {x}")
+
+    opts.data["xyz_fallback_sampler"] = sampler_name
+
+
 def apply_uni_pc_order(p, x, xs):
     opts.data["uni_pc_order"] = min(x, p.steps - 1)
 
@@ -220,6 +228,7 @@ axis_options = [
     AxisOption("Clip skip", int, apply_clip_skip),
     AxisOption("Denoising", float, apply_field("denoising_strength")),
     AxisOptionTxt2Img("Hires upscaler", str, apply_field("hr_upscaler"), choices=lambda: [*shared.latent_upscale_modes, *[x.name for x in shared.sd_upscalers]]),
+    AxisOptionTxt2Img("Fallback latent upscaler sampler", str, apply_fallback, format_value=format_value, confirm=confirm_samplers, choices=lambda: [x.name for x in sd_samplers.samplers]),
     AxisOptionImg2Img("Cond. Image Mask Weight", float, apply_field("inpainting_mask_weight")),
     AxisOption("VAE", str, apply_vae, cost=0.7, choices=lambda: list(sd_vae.vae_dict)),
     AxisOption("Styles", str, apply_styles, choices=lambda: list(shared.prompt_styles.styles)),

From 5148c5b0ad6a5a9522e57952b428f89d29f64375 Mon Sep 17 00:00:00 2001
From: cool-bigdogs-tshirt
 <131823432+cool-bigdogs-tshirt@users.noreply.github.com>
Date: Fri, 28 Apr 2023 22:50:56 -0500
Subject: [PATCH 19/69] fix batching issue

---
 modules/models/diffusion/uni_pc/sampler.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/modules/models/diffusion/uni_pc/sampler.py b/modules/models/diffusion/uni_pc/sampler.py
index 3b468cf3f..3100522ab 100644
--- a/modules/models/diffusion/uni_pc/sampler.py
+++ b/modules/models/diffusion/uni_pc/sampler.py
@@ -47,8 +47,6 @@ class UniPCSampler(object):
                 shared.opts.uni_pc_order+1,
             )
 
-        t = torch.full(t.shape, self.steps).to(t.device)
-
         scheduler_timesteps = np.linspace(
                 0,
                 self.model.num_timesteps-1,
@@ -62,13 +60,17 @@ class UniPCSampler(object):
         latent_timestep = sample_timesteps[:1].repeat(x0.shape[0])
 
         alphas_cumprod = self.alphas_cumprod
-        sqrt_alphas_prod = alphas_cumprod[latent_timestep] ** 0.5
-        sqrt_alphas_prod = sqrt_alphas_prod.flatten()
+        sqrt_alpha_prod = alphas_cumprod[latent_timestep] ** 0.5
+        sqrt_alpha_prod = sqrt_alpha_prod.flatten()
+        while len(sqrt_alpha_prod.shape) < len(x0.shape):
+            sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1)
 
         sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[latent_timestep]) ** 0.5
         sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten()
+        while len(sqrt_one_minus_alpha_prod.shape) < len(x0.shape):
+            sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1)
 
-        return (sqrt_alphas_prod * x0 + sqrt_one_minus_alpha_prod * noise)
+        return (sqrt_alpha_prod * x0 + sqrt_one_minus_alpha_prod * noise)
 
     def decode(self, x_latent, conditioning, t_start, unconditional_guidance_scale=1.0, unconditional_conditioning=None,
                use_original_steps=False, callback=None):

From d51918c68210f9cdecd156932559d8e2773711f9 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 07:37:53 -0400
Subject: [PATCH 20/69] change order of argparse

---
 .../multidiffusion-upscaler-for-automatic1111       |  2 +-
 extensions-builtin/sd-webui-controlnet              |  2 +-
 launch.py                                           |  1 +
 modules/cmd_args.py                                 |  1 +
 setup.py                                            | 13 +++++++------
 5 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/extensions-builtin/multidiffusion-upscaler-for-automatic1111 b/extensions-builtin/multidiffusion-upscaler-for-automatic1111
index f3d79a474..5da750b9d 160000
--- a/extensions-builtin/multidiffusion-upscaler-for-automatic1111
+++ b/extensions-builtin/multidiffusion-upscaler-for-automatic1111
@@ -1 +1 @@
-Subproject commit f3d79a474b9795f07143eaf8104737a403b5fb52
+Subproject commit 5da750b9de930b0e28883423f697c5ea82457c24
diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index 2bc440001..940d4edfb 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit 2bc4400011b38ab7f1d3f27a95897a6cb0c28c2a
+Subproject commit 940d4edfbab1525615b1827a9cb7b7ea21af8a6c
diff --git a/launch.py b/launch.py
index ec9fd514b..bf34ee507 100644
--- a/launch.py
+++ b/launch.py
@@ -14,6 +14,7 @@ from rich import print # pylint: disable=redefined-builtin,wrong-import-order
 
 commandline_args = os.environ.get('COMMANDLINE_ARGS', "")
 sys.argv += shlex.split(commandline_args)
+setup.add_args()
 setup.extensions_preload(force=False)
 setup.parse_args()
 args, _ = modules.cmd_args.parser.parse_known_args()
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index 65d4d4cfe..692f6451f 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -78,6 +78,7 @@ def compatibility_args(opts, args):
     opts.use_old_emphasis_implementation = False
     opts.use_old_karras_scheduler_sigmas = False
     opts.no_dpmpp_sde_batch_determinism = False
+    opts.lora_apply_to_outputs = False
 
     parser.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir)
     args = parser.parse_args()
diff --git a/setup.py b/setup.py
index d368b44f3..3963832dd 100644
--- a/setup.py
+++ b/setup.py
@@ -486,12 +486,9 @@ def check_timestamp():
     return ok
 
 
-def parse_args():
-    # command line args
-    # parser = argparse.ArgumentParser(description = 'Setup for SD WebUI')
-    if vars(parser)['_option_string_actions'].get('--debug', None) is not None:
-        return
-    parser.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
+def add_args():
+    if vars(parser)['_option_string_actions'].get('--debug', None) is None:
+        parser.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
     parser.add_argument('--reset', default = False, action='store_true', help = "Reset main repository to latest version, default: %(default)s")
     parser.add_argument('--upgrade', default = False, action='store_true', help = "Upgrade main repository to latest version, default: %(default)s")
     parser.add_argument('--noupdate', default = False, action='store_true', help = "Skip update of extensions and submodules, default: %(default)s")
@@ -499,6 +496,10 @@ def parse_args():
     parser.add_argument('--skip-extensions', default = False, action='store_true', help = "Skips running individual extension installers, default: %(default)s")
     parser.add_argument('--skip-git', default = False, action='store_true', help = "Skips running all GIT operations, default: %(default)s")
     parser.add_argument('--experimental', default = False, action='store_true', help = "Allow unsupported versions of libraries, default: %(default)s")
+
+
+def parse_args():
+    # command line args
     global args # pylint: disable=global-statement
     args = parser.parse_args()
 

From 4e05d95ee0994260dc50e33219f5808c227d30a1 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 08:58:32 -0400
Subject: [PATCH 21/69] fix prompts from file

---
 .../multidiffusion-upscaler-for-automatic1111 |  2 +-
 html/extra-networks-card.html                 |  7 +--
 modules/processing.py                         | 27 ++-------
 modules/shared.py                             |  2 +-
 modules/ui_common.py                          |  6 +-
 scripts/xyz_grid.py                           | 56 ++++++++-----------
 style.css                                     | 33 +++--------
 7 files changed, 46 insertions(+), 87 deletions(-)

diff --git a/extensions-builtin/multidiffusion-upscaler-for-automatic1111 b/extensions-builtin/multidiffusion-upscaler-for-automatic1111
index 5da750b9d..7253cb449 160000
--- a/extensions-builtin/multidiffusion-upscaler-for-automatic1111
+++ b/extensions-builtin/multidiffusion-upscaler-for-automatic1111
@@ -1 +1 @@
-Subproject commit 5da750b9de930b0e28883423f697c5ea82457c24
+Subproject commit 7253cb449c85e2d2317ba541bf770ef372a807a4
diff --git a/html/extra-networks-card.html b/html/extra-networks-card.html
index cb4720f14..6825a2752 100644
--- a/html/extra-networks-card.html
+++ b/html/extra-networks-card.html
@@ -4,9 +4,9 @@
 	<div class='actions'>
 		<div class='additional'>
 			<ul>
-				<li><a href="#" title="replace preview image with currently selected in gallery" onclick={save_card_preview}>replace preview</a></li>
-				<li><a href="#" title="replace preview description with currently selected in gallery" onclick={save_card_description}>replace description</a></li>
-				<li><a href="#" title="read description" onclick={read_card_description}>read description</a></li>
+				<li><a href="#" title="replace preview image with currently selected in gallery" onclick={save_card_preview}>Replace preview</a></li>
+				<li><a href="#" title="replace preview description with currently selected in gallery" onclick={save_card_description}>Replace description</a></li>
+				<li><a href="#" title="read description" onclick={read_card_description}>Read description</a></li>
 			</ul>
 			<span style="display:none" class='search_term'>{search_term}</span>
 		</div>
@@ -14,4 +14,3 @@
 		<span class='description'>{description}</span>
 	</div>
 </div>
-
diff --git a/modules/processing.py b/modules/processing.py
index a83d9fa12..1c6c36d32 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -971,9 +971,8 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
 
         img2img_sampler_name = self.sampler_name
         force_latent_upscaler = shared.opts.data.get('xyz_fallback_sampler')
-        if self.sampler_name in ['PLMS'] or force_latent_upscaler is not None:
-            # PLMS does not support img2img, use fallback instead
-            img2img_sampler_name = force_latent_upscaler or shared.opts.fallback_sampler
+        if self.sampler_name in ['PLMS'] or (force_latent_upscaler is not None and force_latent_upscaler != 'None'):
+            img2img_sampler_name = force_latent_upscaler or shared.opts.fallback_sampler # PLMS does not support img2img, use fallback instead
         self.sampler = sd_samplers.create_sampler(img2img_sampler_name, self.sd_model)
 
         samples = samples[:, :, self.truncate_y//2:samples.shape[2]-(self.truncate_y+1)//2, self.truncate_x//2:samples.shape[3]-(self.truncate_x+1)//2]
@@ -1026,29 +1025,24 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
         self.image_conditioning = None
 
     def init(self, all_prompts, all_seeds, all_subseeds):
-        if self.sampler_name in ['PLMS', 'UniPC']:  # PLMS/UniPC do not support img2img so we just silently switch to DDIM
-            self.sampler_name = shared.opts.fallback_sampler
+        force_latent_upscaler = shared.opts.data.get('xyz_fallback_sampler')
+        if self.sampler_name in ['PLMS'] or (force_latent_upscaler is not None and force_latent_upscaler != 'None'):
+            self.sampler_name = force_latent_upscaler or shared.opts.fallback_sampler # PLMS does not support img2img, use fallback instead
         self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model)
         crop_region = None
-
         image_mask = self.image_mask
-
         if image_mask is not None:
             image_mask = image_mask.convert('L')
-
             if self.inpainting_mask_invert:
                 image_mask = ImageOps.invert(image_mask)
-
             if self.mask_blur > 0:
                 image_mask = image_mask.filter(ImageFilter.GaussianBlur(self.mask_blur))
-
             if self.inpaint_full_res:
                 self.mask_for_overlay = image_mask
                 mask = image_mask.convert('L')
                 crop_region = masking.get_crop_region(np.array(mask), self.inpaint_full_res_padding)
                 crop_region = masking.expand_crop_region(crop_region, self.width, self.height, mask.width, mask.height)
                 x1, y1, x2, y2 = crop_region
-
                 mask = mask.crop(crop_region)
                 image_mask = images.resize_image(2, mask, self.width, self.height)
                 self.paste_to = (x1, y1, x2-x1, y2-y1)
@@ -1057,42 +1051,31 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
                 np_mask = np.array(image_mask)
                 np_mask = np.clip((np_mask.astype(np.float32)) * 2, 0, 255).astype(np.uint8)
                 self.mask_for_overlay = Image.fromarray(np_mask)
-
             self.overlay_images = []
-
         latent_mask = self.latent_mask if self.latent_mask is not None else image_mask
-
         add_color_corrections = opts.img2img_color_correction and self.color_corrections is None
         if add_color_corrections:
             self.color_corrections = []
         imgs = []
         for img in self.init_images:
             image = images.flatten(img, opts.img2img_background_color)
-
             if crop_region is None and self.resize_mode != 3:
                 image = images.resize_image(self.resize_mode, image, self.width, self.height)
-
             if image_mask is not None:
                 image_masked = Image.new('RGBa', (image.width, image.height))
                 image_masked.paste(image.convert("RGBA").convert("RGBa"), mask=ImageOps.invert(self.mask_for_overlay.convert('L')))
-
                 self.overlay_images.append(image_masked.convert('RGBA'))
-
             # crop_region is not None if we are doing inpaint full res
             if crop_region is not None:
                 image = image.crop(crop_region)
                 image = images.resize_image(2, image, self.width, self.height)
-
             if image_mask is not None:
                 if self.inpainting_fill != 1:
                     image = masking.fill(image, latent_mask)
-
             if add_color_corrections:
                 self.color_corrections.append(setup_color_correction(image))
-
             image = np.array(image).astype(np.float32) / 255.0
             image = np.moveaxis(image, 2, 0)
-
             imgs.append(image)
 
         if len(imgs) == 1:
diff --git a/modules/shared.py b/modules/shared.py
index 5c94bc4ff..c18e31021 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -417,7 +417,7 @@ options_templates.update(options_section(('ui', "Live previews"), {
 
 options_templates.update(options_section(('sampler-params', "Sampler parameters"), {
     "show_samplers": OptionInfo(["Euler a", "UniPC", "DDIM", "DPM++ SDE", "DPM++ SDE", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers()]}),
-    "fallback_sampler": OptionInfo("Euler a", "Fallback sampler if primary sampler is not compatible", gr.Dropdown, lambda: {"choices": [x.name for x in list_samplers()]}),
+    "fallback_sampler": OptionInfo("Euler a", "Fallback sampler if primary sampler is not compatible", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}),
     "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),
     "eta_ddim": OptionInfo(0.0, "Noise multiplier for DDIM (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),
     "ddim_discretize": OptionInfo('uniform', "DDIM discretize img2img", gr.Radio, {"choices": ['uniform', 'quad']}),
diff --git a/modules/ui_common.py b/modules/ui_common.py
index 12908005e..cb9870196 100644
--- a/modules/ui_common.py
+++ b/modules/ui_common.py
@@ -66,10 +66,12 @@ def save_files(js_data, images, do_make_zip, index):
 
         for image_index, filedata in enumerate(images, start_index):
             image = image_from_url_text(filedata)
-
             is_grid = image_index < p.index_of_first_image
             i = 0 if is_grid else (image_index - p.index_of_first_image)
-
+            if len(p.all_seeds) <= i:
+                p.all_seeds.append(p.seed)
+            if len(p.all_prompts) <= i:
+                p.all_prompts.append(p.prompt)
             fullfn, txt_fullfn = modules.images.save_image(image, path, "", seed=p.all_seeds[i], prompt=p.all_prompts[i], extension=extension, info=p.infotexts[image_index], grid=is_grid, p=p, save_to_dirs=save_to_dirs)
 
             filename = os.path.relpath(fullfn, path)
diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py
index 9a5a67241..95fa9ac95 100644
--- a/scripts/xyz_grid.py
+++ b/scripts/xyz_grid.py
@@ -1,26 +1,18 @@
+import re
+import csv
+import random
 from collections import namedtuple
 from copy import copy
 from itertools import permutations, chain
-import random
-import csv
 from io import StringIO
 from PIL import Image
 import numpy as np
-
-import modules.scripts as scripts
 import gradio as gr
-
-from modules import images, paths, sd_samplers, processing, sd_models, sd_vae
-from modules.processing import process_images, Processed, StableDiffusionProcessingTxt2Img
-from modules.shared import opts, cmd_opts, state
+import modules.scripts as scripts
 import modules.shared as shared
-import modules.sd_samplers
-import modules.sd_models
-import modules.sd_vae
-import glob
-import os
-import re
-
+from modules import images, sd_samplers, processing, sd_models, sd_vae
+from modules.processing import process_images, Processed, StableDiffusionProcessingTxt2Img
+from modules.shared import opts, state
 from modules.ui_components import ToolButton
 
 fill_values_symbol = "\U0001f4d2"  # 📒
@@ -83,15 +75,15 @@ def confirm_samplers(p, xs):
 
 
 def apply_checkpoint(p, x, xs):
-    info = modules.sd_models.get_closet_checkpoint_match(x)
+    info = sd_models.get_closet_checkpoint_match(x)
     if info is None:
         raise RuntimeError(f"Unknown checkpoint: {x}")
-    modules.sd_models.reload_model_weights(shared.sd_model, info)
+    sd_models.reload_model_weights(shared.sd_model, info)
 
 
 def confirm_checkpoints(p, xs):
     for x in xs:
-        if modules.sd_models.get_closet_checkpoint_match(x) is None:
+        if sd_models.get_closet_checkpoint_match(x) is None:
             raise RuntimeError(f"Unknown checkpoint: {x}")
 
 
@@ -108,20 +100,20 @@ def apply_upscale_latent_space(p, x, xs):
 
 def find_vae(name: str):
     if name.lower() in ['auto', 'automatic']:
-        return modules.sd_vae.unspecified
+        return sd_vae.unspecified
     if name.lower() == 'none':
         return None
     else:
-        choices = [x for x in sorted(modules.sd_vae.vae_dict, key=lambda x: len(x)) if name.lower().strip() in x.lower()]
+        choices = [x for x in sorted(sd_vae.vae_dict, key=lambda x: len(x)) if name.lower().strip() in x.lower()]
         if len(choices) == 0:
             print(f"No VAE found for {name}; using automatic")
-            return modules.sd_vae.unspecified
+            return sd_vae.unspecified
         else:
-            return modules.sd_vae.vae_dict[choices[0]]
+            return sd_vae.vae_dict[choices[0]]
 
 
 def apply_vae(p, x, xs):
-    modules.sd_vae.reload_vae_weights(shared.sd_model, vae_file=find_vae(x))
+    sd_vae.reload_vae_weights(shared.sd_model, vae_file=find_vae(x))
 
 
 def apply_styles(p: StableDiffusionProcessingTxt2Img, x: str, _):
@@ -341,7 +333,6 @@ def draw_xyz_grid(p, xs, ys, zs, x_labels, y_labels, z_labels, cell, draw_legend
     if draw_legend:
         z_grid = images.draw_grid_annotations(z_grid, sub_grid_size[0], sub_grid_size[1], title_texts, [[images.GridAnnotation()]])
     processed_result.images.insert(0, z_grid)
-    #TODO: Deeper aspects of the program rely on grid info being misaligned between metadata arrays, which is not ideal.
     #processed_result.all_prompts.insert(0, processed_result.all_prompts[0])
     #processed_result.all_seeds.insert(0, processed_result.all_seeds[0])
     processed_result.infotexts.insert(0, processed_result.infotexts[0])
@@ -354,12 +345,12 @@ class SharedSettingsStackHelper(object):
         self.CLIP_stop_at_last_layers = opts.CLIP_stop_at_last_layers
         self.vae = opts.sd_vae
         self.uni_pc_order = opts.uni_pc_order
-  
+
     def __exit__(self, exc_type, exc_value, tb):
         opts.data["sd_vae"] = self.vae
         opts.data["uni_pc_order"] = self.uni_pc_order
-        modules.sd_models.reload_model_weights()
-        modules.sd_vae.reload_vae_weights()
+        sd_models.reload_model_weights()
+        sd_vae.reload_vae_weights()
 
         opts.data["CLIP_stop_at_last_layers"] = self.CLIP_stop_at_last_layers
 
@@ -407,7 +398,7 @@ class Script(scripts.Script):
                 include_sub_grids = gr.Checkbox(label='Include Sub Grids', value=False, elem_id=self.elem_id("include_sub_grids"))
             with gr.Column():
                 margin_size = gr.Slider(label="Grid margins (px)", minimum=0, maximum=500, value=0, step=2, elem_id=self.elem_id("margin_size"))
-        
+
         with gr.Row(variant="compact", elem_id="swap_axes"):
             swap_xy_axes_button = gr.Button(value="Swap X/Y axes", elem_id="xy_grid_swap_axes_button")
             swap_yz_axes_button = gr.Button(value="Swap Y/Z axes", elem_id="yz_grid_swap_axes_button")
@@ -468,7 +459,7 @@ class Script(scripts.Script):
 
     def run(self, p, x_type, x_values, x_values_dropdown, y_type, y_values, y_values_dropdown, z_type, z_values, z_values_dropdown, draw_legend, include_lone_images, include_sub_grids, no_fixed_seeds, margin_size):
         if not no_fixed_seeds:
-            modules.processing.fix_seed(p)
+            processing.fix_seed(p)
 
         if not opts.return_grid:
             p.batch_size = 1
@@ -498,7 +489,7 @@ class Script(scripts.Script):
                         start = int(mc.group(1))
                         end   = int(mc.group(2))
                         num   = int(mc.group(3)) if mc.group(3) is not None else 1
-                        
+
                         valslist_ext += [int(x) for x in np.linspace(start=start, stop=end, num=num).tolist()]
                     else:
                         valslist_ext.append(val)
@@ -520,7 +511,7 @@ class Script(scripts.Script):
                         start = float(mc.group(1))
                         end   = float(mc.group(2))
                         num   = int(mc.group(3)) if mc.group(3) is not None else 1
-                        
+
                         valslist_ext += np.linspace(start=start, stop=end, num=num).tolist()
                     else:
                         valslist_ext.append(val)
@@ -708,13 +699,12 @@ class Script(scripts.Script):
             # Auto-save main and sub-grids:
             grid_count = z_count + 1 if z_count > 1 else 1
             for g in range(grid_count):
-                #TODO: See previous comment about intentional data misalignment.
                 adj_g = g-1 if g > 0 else g
                 images.save_image(processed.images[g], p.outpath_grids, "xyz_grid", info=processed.infotexts[g], extension=opts.grid_format, prompt=processed.all_prompts[adj_g], seed=processed.all_seeds[adj_g], grid=True, p=processed)
 
         if not include_sub_grids:
             # Done with sub-grids, drop all related information:
-            for sg in range(z_count):
+            for _sg in range(z_count):
                 del processed.images[1]
                 del processed.all_prompts[1]
                 del processed.all_seeds[1]
diff --git a/style.css b/style.css
index 55c571d6f..4f5449a61 100644
--- a/style.css
+++ b/style.css
@@ -679,10 +679,6 @@ footer {
     margin-left: 0.5em;
 }
 
-
-.extra-network-cards .card .metadata-button:before, .extra-network-thumbs .card .metadata-button:before{
-    content: "🛈";
-}
 .extra-network-cards .card .metadata-button, .extra-network-thumbs .card .metadata-button{
     display: none;
     position: absolute;
@@ -696,11 +692,11 @@ footer {
 .extra-network-cards .card:hover .metadata-button, .extra-network-thumbs .card:hover .metadata-button{
     display: inline-block;
 }
+
 .extra-network-cards .card .metadata-button:hover, .extra-network-thumbs .card .metadata-button:hover{
     color: red;
 }
 
-
 .extra-network-thumbs {
     display: flex;
     flex-flow: row wrap;
@@ -708,8 +704,9 @@ footer {
 }
 
 .extra-network-thumbs .card {
-    height: 6em;
-    width: 6em;
+    display: inline-block;
+    height: 9em;
+    width: 9em;
     cursor: pointer;
     background-image: url('./file=html/card-no-preview.png');
     background-size: cover;
@@ -717,23 +714,13 @@ footer {
     position: relative;
 }
 
-.extra-network-thumbs .card:hover .additional a {
-    display: inline-block;
+.extra-network-thumbs .card .additional, .extra-network-thumbs .card .additional {
+    white-space: nowrap;
+    overflow: hidden;
 }
 
-.extra-network-thumbs .actions .additional a {
-    background-image: url('./file=html/image-update.svg');
-    background-repeat: no-repeat;
-    background-size: cover;
-    background-position: center center;
-    position: absolute;
-    top: 0;
-    left: 0;
-    width: 24px;
-    height: 24px;
-    display: none;
-    font-size: 0;
-    text-align: -9999;
+.extra-network-thumbs .card:hover .additional a {
+    display: inline-block;
 }
 
 .extra-network-thumbs .actions .name {
@@ -762,12 +749,10 @@ footer {
     box-shadow: 0 0 5px rgba(128, 128, 128, 0.5);
     border-radius: 0.2em;
     position: relative;
-
     background-size: auto 100%;
     background-position: center;
     overflow: hidden;
     cursor: pointer;
-
     background-image: url('./file=html/card-no-preview.png')
 }
 

From 6831b033808b6b4b7f341be638bafcf05e1ce845 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 09:32:06 -0400
Subject: [PATCH 22/69] fix cards previews

---
 .../multidiffusion-upscaler-for-automatic1111 |   2 +-
 html/extra-networks-card.html                 |   4 +-
 style.css                                     | 353 +++---------------
 3 files changed, 62 insertions(+), 297 deletions(-)

diff --git a/extensions-builtin/multidiffusion-upscaler-for-automatic1111 b/extensions-builtin/multidiffusion-upscaler-for-automatic1111
index 7253cb449..6931b89cb 160000
--- a/extensions-builtin/multidiffusion-upscaler-for-automatic1111
+++ b/extensions-builtin/multidiffusion-upscaler-for-automatic1111
@@ -1 +1 @@
-Subproject commit 7253cb449c85e2d2317ba541bf770ef372a807a4
+Subproject commit 6931b89cb4507c7dc8fa81ac36c2c19d0691c44e
diff --git a/html/extra-networks-card.html b/html/extra-networks-card.html
index 6825a2752..3cf6e2836 100644
--- a/html/extra-networks-card.html
+++ b/html/extra-networks-card.html
@@ -4,8 +4,8 @@
 	<div class='actions'>
 		<div class='additional'>
 			<ul>
-				<li><a href="#" title="replace preview image with currently selected in gallery" onclick={save_card_preview}>Replace preview</a></li>
-				<li><a href="#" title="replace preview description with currently selected in gallery" onclick={save_card_description}>Replace description</a></li>
+				<li><a href="#" title="replace preview image with current selection" onclick={save_card_preview}>Replace preview</a></li>
+				<li><a href="#" title="replace preview description with current selection" onclick={save_card_description}>Replace description</a></li>
 				<li><a href="#" title="read description" onclick={read_card_description}>Read description</a></li>
 			</ul>
 			<span style="display:none" class='search_term'>{search_term}</span>
diff --git a/style.css b/style.css
index 4f5449a61..996ad15d2 100644
--- a/style.css
+++ b/style.css
@@ -1,165 +1,36 @@
-
-/* general gradio fixes */
-
-:root, .dark{
-    --checkbox-label-gap: 0.25em 0.1em;
-    --section-header-text-size: 12pt;
-    --block-background-fill: transparent;
-}
-
-.block.padded:not(.gradio-accordion) {
-    padding: 0 !important;
-}
-
-div.gradio-container{
-    max-width: unset !important;
-}
-
-.hidden{
-    display: none;
-}
-
-.compact{
-    background: transparent !important;
-    padding: 0 !important;
-}
-
-div.form{
-    border-width: 0;
-    box-shadow: none;
-    background: transparent;
-    overflow: visible;
-    gap: 0.5em;
-}
-
-.block.gradio-dropdown,
-.block.gradio-slider,
-.block.gradio-checkbox,
-.block.gradio-textbox,
-.block.gradio-radio,
-.block.gradio-checkboxgroup,
-.block.gradio-number,
-.block.gradio-colorpicker
-{
-    border-width: 0 !important;
-    box-shadow: none !important;
-}
-
-.gap.compact{
-    padding: 0;
-    gap: 0.2em 0;
-}
-
-div.compact{
-    gap: 1em;
-}
-
-.gradio-dropdown label span:not(.has-info),
-.gradio-textbox label span:not(.has-info),
-.gradio-number label span:not(.has-info)
-{
-    margin-bottom: 0;
-}
-
-.gradio-dropdown ul.options{
-    z-index: 3000;
-    min-width: fit-content;
-    max-width: inherit;
-    white-space: nowrap;
-}
-
-.gradio-dropdown ul.options li.item {
-    padding: 0.05em 0;
-}
-
-.gradio-dropdown ul.options li.item:not(:has(.hide)) {
-    background-color: var(--neutral-100);
-}
-
-.dark .gradio-dropdown ul.options li.item:not(:has(.hide)) {
-    background-color: var(--neutral-900);
-}
-
-.gradio-dropdown div.wrap.wrap.wrap.wrap{
-    box-shadow: 0 1px 2px 0 rgba(0, 0, 0, 0.05);
-}
-
-.gradio-dropdown:not(.multiselect) .wrap-inner.wrap-inner.wrap-inner{
-    flex-wrap: unset;
-}
-
-.gradio-dropdown .single-select{
-    white-space: nowrap;
-    overflow: hidden;
-}
-
-.gradio-dropdown .token-remove.remove-all.remove-all{
-    display: none;
-}
-
-.gradio-dropdown.multiselect .token-remove.remove-all.remove-all{
-    display: flex;
-}
-
-.gradio-slider input[type="number"]{
-    width: 6em;
-}
-
-.block.gradio-checkbox {
-    margin: 0.75em 1.5em 0 0;
-}
-
-.gradio-html div.wrap{
-    height: 100%;
-}
-div.gradio-html.min{
-    min-height: 0;
-}
-
-.block.gradio-gallery{
-    background: var(--input-background-fill);
-}
-
-.gradio-container .prose a, .gradio-container .prose a:visited{
-    color: unset;
-    text-decoration: none;
-}
-
-
+:root, .dark{ --checkbox-label-gap: 0.25em 0.1em; --section-header-text-size: 12pt; --block-background-fill: transparent;}
+.block.padded:not(.gradio-accordion) { padding: 0 !important; }
+div.gradio-container{ max-width: unset !important; }
+.hidden{ display: none; }
+.compact{ background: transparent !important; padding: 0 !important; }
+div.form{ border-width: 0; box-shadow: none; background: transparent; overflow: visible; gap: 0.5em; }
+.block.gradio-dropdown, .block.gradio-slider, .block.gradio-checkbox, .block.gradio-textbox, .block.gradio-radio, .block.gradio-checkboxgroup, .block.gradio-number, .block.gradio-colorpicker { border-width: 0 !important; box-shadow: none !important;}
+.gap.compact{ padding: 0; gap: 0.2em 0; }
+div.compact{ gap: 1em; }
+.gradio-dropdown label span:not(.has-info), .gradio-textbox label span:not(.has-info), .gradio-number label span:not(.has-info) { margin-bottom: 0; }
+.gradio-dropdown ul.options{ z-index: 3000; min-width: fit-content; max-width: inherit; white-space: nowrap; }
+.gradio-dropdown ul.options li.item { padding: 0.05em 0; }
+.gradio-dropdown ul.options li.item:not(:has(.hide)) { background-color: var(--neutral-100); }
+.dark .gradio-dropdown ul.options li.item:not(:has(.hide)) { background-color: var(--neutral-900); }
+.gradio-dropdown div.wrap.wrap.wrap.wrap{ box-shadow: 0 1px 2px 0 rgba(0, 0, 0, 0.05); }
+.gradio-dropdown:not(.multiselect) .wrap-inner.wrap-inner.wrap-inner{ flex-wrap: unset; }
+.gradio-dropdown .single-select{ white-space: nowrap; overflow: hidden; }
+.gradio-dropdown .token-remove.remove-all.remove-all{ display: none; }
+.gradio-dropdown.multiselect .token-remove.remove-all.remove-all{ display: flex; }
+.gradio-slider input[type="number"]{ width: 6em; }
+.block.gradio-checkbox { margin: 0.75em 1.5em 0 0; }
+.gradio-html div.wrap{ height: 100%; }
+div.gradio-html.min{ min-height: 0; }
+.block.gradio-gallery{ background: var(--input-background-fill); }
+.gradio-container .prose a, .gradio-container .prose a:visited{ color: unset; text-decoration: none; }
 
 /* general styled components */
-
-.gradio-button.tool{
-    max-width: 2.2em;
-    min-width: 2.2em !important;
-    height: 2.4em;
-    align-self: end;
-    line-height: 1em;
-    border-radius: 0.5em;
-}
-
-.gradio-button.secondary-down{
-    background: var(--button-secondary-background-fill);
-    color: var(--button-secondary-text-color);
-}
-.gradio-button.secondary-down, .gradio-button.secondary-down:hover{
-    box-shadow: 1px 1px 1px rgba(0,0,0,0.25) inset, 0px 0px 3px rgba(0,0,0,0.15) inset;
-}
-.gradio-button.secondary-down:hover{
-    background: var(--button-secondary-background-fill-hover);
-    color: var(--button-secondary-text-color-hover);
-}
-
-.checkboxes-row{
-    margin-bottom: 0.5em;
-    margin-left: 0em;
-}
-.checkboxes-row > div{
-    flex: 0;
-    white-space: nowrap;
-    min-width: auto;
-}
-
+.gradio-button.tool{ max-width: 2.2em;  min-width: 2.2em !important; height: 2.4em; align-self: end; line-height: 1em; border-radius: 0.5em; }
+.gradio-button.secondary-down{ background: var(--button-secondary-background-fill); color: var(--button-secondary-text-color); }
+.gradio-button.secondary-down, .gradio-button.secondary-down:hover{ box-shadow: 1px 1px 1px rgba(0,0,0,0.25) inset, 0px 0px 3px rgba(0,0,0,0.15) inset; }
+.gradio-button.secondary-down:hover{ background: var(--button-secondary-background-fill-hover); color: var(--button-secondary-text-color-hover); }
+.checkboxes-row{ margin-bottom: 0.5em; margin-left: 0em; }
+.checkboxes-row > div{ flex: 0; white-space: nowrap; min-width: auto; }
 button.custom-button{
     border-radius: var(--button-large-radius);
     padding: var(--button-large-padding);
@@ -176,9 +47,7 @@ button.custom-button{
     text-align: center;
 }
 
-
 /* txt2img/img2img specific */
-
 .block.token-counter{
     position: absolute;
     display: inline-block;
@@ -201,13 +70,8 @@ button.custom-button{
     border: 2px solid rgba(255,0,0,0.4) !important;
 }
 
-.block.token-counter div{
-    display: inline;
-}
-
-.block.token-counter span{
-    padding: 0.1em 0.75em;
-}
+.block.token-counter div{ display: inline; }
+.block.token-counter span{ padding: 0.1em 0.75em; }
 
 [id$=_subseed_show]{
     min-width: auto !important;
@@ -642,42 +506,14 @@ footer {
 }
 
 /* extra networks UI */
-
-.extra-networks > div > [id *= '_extra_']{
-    margin: 0.3em;
-}
-
-.extra-network-subdirs{
-    padding: 0.2em 0.35em;
-}
-
-.extra-network-subdirs button{
-    margin: 0 0.15em;
-}
-.extra-networks .tab-nav .search{
-    display: inline-block;
-    max-width: 16em;
-    margin: 0.3em;
-    align-self: center;
-    width: 16em;
-}
-
-#txt2img_extra_view, #img2img_extra_view {
-    width: auto;
-}
-
-.extra-network-cards .nocards, .extra-network-thumbs .nocards{
-    margin: 1.25em 0.5em 0.5em 0.5em;
-}
-
-.extra-network-cards .nocards h1, .extra-network-thumbs .nocards h1{
-    font-size: 1.5em;
-    margin-bottom: 1em;
-}
-
-.extra-network-cards .nocards li, .extra-network-thumbs .nocards li{
-    margin-left: 0.5em;
-}
+.extra-networks > div > [id *= '_extra_']{ margin: 0.3em; }
+.extra-network-subdirs{ padding: 0.2em 0.35em; }
+.extra-network-subdirs button{ margin: 0 0.15em; }
+.extra-networks .tab-nav .search{ display: inline-block; max-width: 16em; margin: 0.3em; align-self: center; width: 16em; }
+#txt2img_extra_view, #img2img_extra_view { width: auto; }
+.extra-network-cards .nocards, .extra-network-thumbs .nocards{ margin: 1.25em 0.5em 0.5em 0.5em; }
+.extra-network-cards .nocards h1, .extra-network-thumbs .nocards h1{ font-size: 1.5em; margin-bottom: 1em; }
+.extra-network-cards .nocards li, .extra-network-thumbs .nocards li{ margin-left: 0.5em; }
 
 .extra-network-cards .card .metadata-button, .extra-network-thumbs .card .metadata-button{
     display: none;
@@ -689,19 +525,9 @@ footer {
     font-size: 22pt;
     width: 1.5em;
 }
-.extra-network-cards .card:hover .metadata-button, .extra-network-thumbs .card:hover .metadata-button{
-    display: inline-block;
-}
-
-.extra-network-cards .card .metadata-button:hover, .extra-network-thumbs .card .metadata-button:hover{
-    color: red;
-}
-
-.extra-network-thumbs {
-    display: flex;
-    flex-flow: row wrap;
-    gap: 10px;
-}
+.extra-network-cards .card:hover .metadata-button, .extra-network-thumbs .card:hover .metadata-button{ display: inline-block; }
+.extra-network-thumbs { display: flex; flex-flow: row wrap; gap: 10px; }
+.extra-network-cards .card .additional a:hover, .extra-network-thumbs .card .additional a:hover { color: darkorange }
 
 .extra-network-thumbs .card {
     display: inline-block;
@@ -714,15 +540,8 @@ footer {
     position: relative;
 }
 
-.extra-network-thumbs .card .additional, .extra-network-thumbs .card .additional {
-    white-space: nowrap;
-    overflow: hidden;
-}
-
-.extra-network-thumbs .card:hover .additional a {
-    display: inline-block;
-}
-
+.extra-network-cards .card .additional, .extra-network-thumbs .card .additional { white-space: nowrap; overflow: hidden; }
+.extra-network-thumbs .card:hover .additional a { display: inline-block; }
 .extra-network-thumbs .actions .name {
     position: absolute;
     bottom: 0;
@@ -736,11 +555,7 @@ footer {
     color: white;
 }
 
-.extra-network-thumbs .card:hover .actions .name {
-    white-space: normal;
-    word-break: break-all;
-}
-
+.extra-network-thumbs .card:hover .actions .name { white-space: normal; word-break: break-all; }
 .extra-network-cards .card{
     display: inline-block;
     margin: 0.5em;
@@ -756,13 +571,8 @@ footer {
     background-image: url('./file=html/card-no-preview.png')
 }
 
-.extra-network-cards .card:hover{
-    box-shadow: 0 0 2px 0.3em rgba(0, 128, 255, 0.35);
-}
-
-.extra-network-cards .card .actions .additional{
-    display: none;
-}
+.extra-network-cards .card:hover { box-shadow: 0 0 2px 0.3em rgba(0, 128, 255, 0.35); }
+.extra-network-cards .card .actions .additional, .extra-network-thumbs .card .actions .additional{ display: none; }
 
 .extra-network-cards .card .actions{
     position: absolute;
@@ -775,58 +585,13 @@ footer {
     text-shadow: 0 0 0.2em black;
 }
 
-.extra-network-cards .card .actions *{
-    color: white;
-}
-
-.extra-network-cards .card .actions:hover{
-    box-shadow: 0 0 0.75em 0.75em rgba(0,0,0,0.5) !important;
-}
-
-.extra-network-cards .card .actions .name{
-    font-size: 1.7em;
-    font-weight: bold;
-    line-break: anywhere;
-}
-
-.extra-network-cards .card .actions .description {
-    display: block;
-    max-height: 3em;
-    white-space: pre-wrap;
-    line-height: 1.1;
-}
-
-.extra-network-cards .card .actions .description:hover {
-    max-height: none;
-}
-
-.extra-network-cards .card .actions:hover .additional{
-    display: block;
-}
-
-.extra-network-cards .card ul{
-    margin: 0.25em 0 0.75em 0.25em;
-    cursor: unset;
-}
-
-.extra-network-cards .card ul a{
-    cursor: pointer;
-}
-
-.extra-network-cards .card ul a:hover{
-    color: red;
-}
-
-.theme-preview {
-    display: none;
-    position: fixed;
-    border: 4px solid var(--neutral-600);
-    box-shadow: 2px 2px 2px 2px var(--neutral-700);
-    top: 0;
-    bottom: 0;
-    left: 0;
-    right: 0;
-    margin: auto;
-    max-width: 75vw;
-    z-index: 999;
-}
+.extra-network-cards .card .actions *{ color: white; }
+.extra-network-cards .card .actions:hover { box-shadow: 0 0 0.75em 0.75em rgba(0,0,0,0.5) !important; }
+.extra-network-cards .card .actions .name { font-size: 1.7em; font-weight: bold; line-break: anywhere; }
+.extra-network-cards .card .actions .description { display: block; max-height: 3em; white-space: pre-wrap; line-height: 1.1; }
+.extra-network-cards .card .actions .description:hover { max-height: none; }
+.extra-network-cards .card .actions:hover .additional, .extra-network-thumbs .card:hover .additional{ display: block; }
+.extra-network-cards .card ul{ margin: 0.25em 0 0.75em 0.25em; cursor: unset; }
+.extra-network-cards .card ul a{ cursor: pointer; }
+.extra-network-cards .card ul a:hover{ color: red; }
+.theme-preview { display: none; position: fixed; border: 4px solid var(--neutral-600); box-shadow: 2px 2px 2px 2px var(--neutral-700); top: 0; bottom: 0; left: 0; right: 0; margin: auto; max-width: 75vw; z-index: 999; }

From 9975f819a0e6de41a26b007ab1320c94e179cccd Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 10:07:09 -0400
Subject: [PATCH 23/69] xyz improvements

---
 scripts/xyz_grid.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py
index 95fa9ac95..748fe7bbc 100644
--- a/scripts/xyz_grid.py
+++ b/scripts/xyz_grid.py
@@ -390,14 +390,12 @@ class Script(scripts.Script):
                     fill_z_button = ToolButton(value=fill_values_symbol, elem_id="xyz_grid_fill_z_tool_button", visible=False)
 
         with gr.Row(variant="compact", elem_id="axis_options"):
-            with gr.Column():
-                draw_legend = gr.Checkbox(label='Draw legend', value=True, elem_id=self.elem_id("draw_legend"))
-                no_fixed_seeds = gr.Checkbox(label='Keep -1 for seeds', value=False, elem_id=self.elem_id("no_fixed_seeds"))
-            with gr.Column():
-                include_lone_images = gr.Checkbox(label='Include Sub Images', value=False, elem_id=self.elem_id("include_lone_images"))
-                include_sub_grids = gr.Checkbox(label='Include Sub Grids', value=False, elem_id=self.elem_id("include_sub_grids"))
-            with gr.Column():
-                margin_size = gr.Slider(label="Grid margins (px)", minimum=0, maximum=500, value=0, step=2, elem_id=self.elem_id("margin_size"))
+            draw_legend = gr.Checkbox(label='Draw legend', value=True, elem_id=self.elem_id("draw_legend"))
+            no_fixed_seeds = gr.Checkbox(label='Keep -1 for seeds', value=False, elem_id=self.elem_id("no_fixed_seeds"))
+            include_lone_images = gr.Checkbox(label='Include Sub Images', value=False, elem_id=self.elem_id("include_lone_images"))
+            include_sub_grids = gr.Checkbox(label='Include Sub Grids', value=False, elem_id=self.elem_id("include_sub_grids"))
+        with gr.Row(variant="compact", elem_id="axis_options"):
+            margin_size = gr.Slider(label="Grid margins (px)", minimum=0, maximum=500, value=0, step=2, elem_id=self.elem_id("margin_size"))
 
         with gr.Row(variant="compact", elem_id="swap_axes"):
             swap_xy_axes_button = gr.Button(value="Swap X/Y axes", elem_id="xy_grid_swap_axes_button")
@@ -428,6 +426,9 @@ class Script(scripts.Script):
             current_values = axis_values_dropdown
             if has_choices:
                 choices = choices()
+                if len(choices) > 12:
+                    has_choices = False
+            if has_choices:
                 if isinstance(current_values,str):
                     current_values = current_values.split(",")
                 current_values = list(filter(lambda x: x in choices, current_values))

From d793afbd03f3ecadedac5156850c18bc22630b82 Mon Sep 17 00:00:00 2001
From: Aurora <46530683+AuwowaUwU@users.noreply.github.com>
Date: Sat, 29 Apr 2023 16:53:41 +0200
Subject: [PATCH 24/69] Remove hardcoded disabling of grids in file script

---
 scripts/prompts_from_file.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/prompts_from_file.py b/scripts/prompts_from_file.py
index 027e6539f..fe30d4b07 100644
--- a/scripts/prompts_from_file.py
+++ b/scripts/prompts_from_file.py
@@ -130,8 +130,6 @@ class Script(scripts.Script):
         lines = [x.strip() for x in prompt_txt.splitlines()]
         lines = [x for x in lines if len(x) > 0]
 
-        p.do_not_save_grid = True
-
         job_count = 0
         jobs = []
 

From c128c0770b31ccb0cf8bc788793f0a4b4f1f6244 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Sun, 30 Apr 2023 01:15:47 +0900
Subject: [PATCH 25/69] Load libatiadlxx.so on Linux systems.

---
 modules/dml/optimizer/amd/driver/atiadlxx_apis.py | 6 +++++-
 setup.py                                          | 1 +
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/modules/dml/optimizer/amd/driver/atiadlxx_apis.py b/modules/dml/optimizer/amd/driver/atiadlxx_apis.py
index fef70b543..25e6390ef 100644
--- a/modules/dml/optimizer/amd/driver/atiadlxx_apis.py
+++ b/modules/dml/optimizer/amd/driver/atiadlxx_apis.py
@@ -1,7 +1,11 @@
 import ctypes as C
+from platform import platform
 from .atiadlxx_structures import *
 
-atiadlxx = C.WinDLL("atiadlxx.dll")
+if 'Windows' in platform():
+    atiadlxx = C.WinDLL("atiadlxx.dll")
+else:
+    atiadlxx = C.CDLL("libatiadlxx.so") # Not tested on Linux system. But will be supported.
 
 ADL_MAIN_MALLOC_CALLBACK = C.CFUNCTYPE(C.c_void_p, C.c_int)
 ADL_MAIN_FREE_CALLBACK = C.CFUNCTYPE(None, C.POINTER(C.c_void_p))
diff --git a/setup.py b/setup.py
index 615ac49d4..c143924a7 100644
--- a/setup.py
+++ b/setup.py
@@ -229,6 +229,7 @@ def check_torch():
                 log.info(f'Torch backend: DirectML ({version})')
                 for i in range(0, torch_directml.device_count()):
                     log.info(f'Torch detected GPU: {torch_directml.device_name(i)}')
+                log.info(f'DirectML default device: {torch_directml.device_name(torch_directml.default_device())}')
             except:
                 log.warning("Torch repoorts CUDA not available")
     except Exception as e:

From 9d8d57a51a415aecacd4bff6f2d8ade3d1ecec01 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Sun, 30 Apr 2023 01:34:40 +0900
Subject: [PATCH 26/69] Sync submodules.

---
 extensions-builtin/multidiffusion-upscaler-for-automatic1111 | 2 +-
 extensions-builtin/sd-webui-controlnet                       | 2 +-
 extensions-builtin/seed_travel                               | 2 +-
 extensions-builtin/stable-diffusion-webui-images-browser     | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/extensions-builtin/multidiffusion-upscaler-for-automatic1111 b/extensions-builtin/multidiffusion-upscaler-for-automatic1111
index f3d79a474..6931b89cb 160000
--- a/extensions-builtin/multidiffusion-upscaler-for-automatic1111
+++ b/extensions-builtin/multidiffusion-upscaler-for-automatic1111
@@ -1 +1 @@
-Subproject commit f3d79a474b9795f07143eaf8104737a403b5fb52
+Subproject commit 6931b89cb4507c7dc8fa81ac36c2c19d0691c44e
diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index c5fbfc31d..940d4edfb 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit c5fbfc31d002ff83bc692e1f24b7b1c9183dbe72
+Subproject commit 940d4edfbab1525615b1827a9cb7b7ea21af8a6c
diff --git a/extensions-builtin/seed_travel b/extensions-builtin/seed_travel
index ffe0553c5..4bc8b2f10 160000
--- a/extensions-builtin/seed_travel
+++ b/extensions-builtin/seed_travel
@@ -1 +1 @@
-Subproject commit ffe0553c59e91067ebf1e4fc7ad85ca9c870bf57
+Subproject commit 4bc8b2f10d5c12958f48b67ad23fb445aff074df
diff --git a/extensions-builtin/stable-diffusion-webui-images-browser b/extensions-builtin/stable-diffusion-webui-images-browser
index 704e42c10..2c988c08c 160000
--- a/extensions-builtin/stable-diffusion-webui-images-browser
+++ b/extensions-builtin/stable-diffusion-webui-images-browser
@@ -1 +1 @@
-Subproject commit 704e42c10d01e6c6965493ec956a82bb8fc2da51
+Subproject commit 2c988c08c7fc2f1c0f572bc4209f0baa1fac4fee

From a41cf2bd84c4de166e0f6ea4871f72e78a1cee58 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Sun, 30 Apr 2023 01:36:55 +0900
Subject: [PATCH 27/69] Sync submodules.

---
 modules/lora | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/lora b/modules/lora
index ac4935bf7..d52c524fc 160000
--- a/modules/lora
+++ b/modules/lora
@@ -1 +1 @@
-Subproject commit ac4935bf79942f966d7b6578e8fbb9ee5f12d4ad
+Subproject commit d52c524fc2942c053cf37c648188502a3a26df1b

From cd580866a7f82e6ace975cff80f1f46586a2988e Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 12:37:33 -0400
Subject: [PATCH 28/69] fix awalys on script args

---
 extensions-builtin/sd-webui-controlnet |   2 +-
 modules/api/api.py                     | 118 +++++++++----------------
 modules/api/models.py                  |  36 ++++----
 modules/cmd_args.py                    |   2 +-
 modules/img2img.py                     |   2 +-
 modules/processing.py                  |   1 +
 modules/scripts.py                     |  28 +++---
 modules/txt2img.py                     |   2 +-
 setup.py                               |   2 +-
 9 files changed, 77 insertions(+), 116 deletions(-)

diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index 940d4edfb..d520e9da0 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit 940d4edfbab1525615b1827a9cb7b7ea21af8a6c
+Subproject commit d520e9da0014073162817ceb1eae241bccba67b9
diff --git a/modules/api/api.py b/modules/api/api.py
index 976b53b57..0717edfaf 100644
--- a/modules/api/api.py
+++ b/modules/api/api.py
@@ -17,7 +17,7 @@ from gradio.processing_utils import decode_base64_to_file
 # from gradio_client.utils import decode_base64_to_file
 
 from modules import errors, shared, sd_samplers, deepbooru, sd_hijack, images, scripts, ui, postprocessing
-from modules.api.models import *
+from modules.api.models import * # pylint: disable=unused-wildcard-import, wildcard-import
 from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, process_images
 from modules.textual_inversion.textual_inversion import create_embedding, train_embedding
 from modules.textual_inversion.preprocess import preprocess
@@ -32,20 +32,19 @@ errors.install()
 def upscaler_to_index(name: str):
     try:
         return [x.name.lower() for x in shared.sd_upscalers].index(name.lower())
-    except:
-        raise HTTPException(status_code=400, detail=f"Invalid upscaler, needs to be one of these: {' , '.join([x.name for x in sd_upscalers])}")
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Invalid upscaler, needs to be one of these: {' , '.join([x.name for x in sd_upscalers])}") from e
 
 def script_name_to_index(name, scripts_list):
     try:
         return [script.title().lower() for script in scripts_list].index(name.lower())
-    except:
-        raise HTTPException(status_code=422, detail=f"Script '{name}' not found")
+    except Exception as e:
+        raise HTTPException(status_code=422, detail=f"Script '{name}' not found") from e
 
 def validate_sampler_name(name):
     config = sd_samplers.all_samplers_map.get(name, None)
     if config is None:
         raise HTTPException(status_code=404, detail="Sampler not found")
-
     return name
 
 def setUpscalers(req: dict):
@@ -60,20 +59,19 @@ def decode_base64_to_image(encoding):
     try:
         image = Image.open(BytesIO(base64.b64decode(encoding)))
         return image
-    except Exception:
-        raise HTTPException(status_code=500, detail="Invalid encoded image")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail="Invalid encoded image") from e
 
 def encode_pil_to_base64(image):
     with io.BytesIO() as output_bytes:
-
         if opts.samples_format.lower() == 'png':
             use_metadata = False
-            metadata = PngImagePlugin.PngInfo()
-            for key, value in image.info.items():
-                if isinstance(key, str) and isinstance(value, str):
-                    metadata.add_text(key, value)
+            encoded_metadata = PngImagePlugin.PngInfo()
+            for k, v in image.info.items():
+                if isinstance(k, str) and isinstance(v, str):
+                    encoded_metadata.add_text(k, v)
                     use_metadata = True
-            image.save(output_bytes, format="PNG", pnginfo=(metadata if use_metadata else None), quality=opts.jpeg_quality)
+            image.save(output_bytes, format="PNG", pnginfo=(encoded_metadata if use_metadata else None), quality=opts.jpeg_quality)
 
         elif opts.samples_format.lower() in ("jpg", "jpeg", "webp"):
             parameters = image.info.get('parameters', None)
@@ -84,12 +82,9 @@ def encode_pil_to_base64(image):
                 image.save(output_bytes, format="JPEG", exif = exif_bytes, quality=opts.jpeg_quality)
             else:
                 image.save(output_bytes, format="WEBP", exif = exif_bytes, quality=opts.jpeg_quality)
-
         else:
             raise HTTPException(status_code=500, detail="Invalid image format")
-
         bytes_data = output_bytes.getvalue()
-
     return base64.b64encode(bytes_data)
 
 
@@ -100,7 +95,6 @@ class Api:
             for auth in shared.cmd_opts.api_auth.split(","):
                 user, password = auth.split(":")
                 self.credentials[user] = password
-
         self.router = APIRouter()
         self.app = app
         self.queue_lock = queue_lock
@@ -135,7 +129,6 @@ class Api:
         self.add_api_route("/sdapi/v1/unload-checkpoint", self.unloadapi, methods=["POST"])
         self.add_api_route("/sdapi/v1/reload-checkpoint", self.reloadapi, methods=["POST"])
         self.add_api_route("/sdapi/v1/scripts", self.get_scripts_list, methods=["GET"], response_model=ScriptsList)
-
         self.default_script_arg_txt2img = []
         self.default_script_arg_img2img = []
 
@@ -148,13 +141,11 @@ class Api:
         if credentials.username in self.credentials:
             if compare_digest(credentials.password, self.credentials[credentials.username]):
                 return True
-
         raise HTTPException(status_code=401, detail="Incorrect username or password", headers={"WWW-Authenticate": "Basic"})
 
     def get_selectable_script(self, script_name, script_runner):
         if script_name is None or script_name == "":
             return None, None
-
         script_idx = script_name_to_index(script_name, script_runner.selectable_scripts)
         script = script_runner.selectable_scripts[script_idx]
         return script, script_idx
@@ -162,13 +153,11 @@ class Api:
     def get_scripts_list(self):
         t2ilist = [str(title.lower()) for title in scripts.scripts_txt2img.titles]
         i2ilist = [str(title.lower()) for title in scripts.scripts_img2img.titles]
-
         return ScriptsList(txt2img = t2ilist, img2img = i2ilist)
 
     def get_script(self, script_name, script_runner):
         if script_name is None or script_name == "":
             return None, None
-
         script_idx = script_name_to_index(script_name, script_runner.scripts)
         return script_runner.scripts[script_idx]
 
@@ -192,25 +181,17 @@ class Api:
                     script_args[script.args_from:script.args_to] = ui_default_values
         return script_args
 
-    def init_script_args(self, request, default_script_args, selectable_scripts, selectable_idx, script_runner):
+    def init_script_args(self, p, request, default_script_args, script_runner):
         script_args = default_script_args.copy()
-        # position 0 in script_arg is the idx+1 of the selectable script that is going to be run when using scripts.scripts_*2img.run()
-        if selectable_scripts:
-            script_args[selectable_scripts.args_from:selectable_scripts.args_to] = request.script_args
-            script_args[0] = selectable_idx + 1
-
-        # Now check for always on scripts
         if request.alwayson_scripts and (len(request.alwayson_scripts) > 0):
             for alwayson_script_name in request.alwayson_scripts.keys():
                 alwayson_script = self.get_script(alwayson_script_name, script_runner)
                 if alwayson_script is None:
                     raise HTTPException(status_code=422, detail=f"always on script {alwayson_script_name} not found")
-                # Selectable script in always on script param check
                 if not alwayson_script.alwayson:
-                    raise HTTPException(status_code=422, detail=f"Cannot have a selectable script in the always on scripts params")
-                # always on script with no arg should always run so you don't really need to add them to the requests
+                    raise HTTPException(status_code=422, detail="Cannot have a selectable script in the always on scripts params")
                 if "args" in request.alwayson_scripts[alwayson_script_name]:
-                    script_args[alwayson_script.args_from:alwayson_script.args_to] = request.alwayson_scripts[alwayson_script_name]["args"]
+                    p.per_script_args[alwayson_script.title()] = request.alwayson_scripts[alwayson_script_name]["args"] + script_args
         return script_args
 
     def text2imgapi(self, txt2imgreq: StableDiffusionTxt2ImgProcessingAPI):
@@ -220,8 +201,7 @@ class Api:
             ui.create_ui()
         if not self.default_script_arg_txt2img:
             self.default_script_arg_txt2img = self.init_default_script_args(script_runner)
-        selectable_scripts, selectable_script_idx = self.get_selectable_script(txt2imgreq.script_name, script_runner)
-
+        selectable_scripts, _selectable_script_idx = self.get_selectable_script(txt2imgreq.script_name, script_runner)
         populate = txt2imgreq.copy(update={  # Override __init__ params
             "sampler_name": validate_sampler_name(txt2imgreq.sampler_name or txt2imgreq.sampler_index),
             "do_not_save_samples": not txt2imgreq.save_images,
@@ -229,14 +209,10 @@ class Api:
         })
         if populate.sampler_name:
             populate.sampler_index = None  # prevent a warning later on
-
         args = vars(populate)
         args.pop('script_name', None)
         args.pop('script_args', None) # will refeed them to the pipeline directly after initializing them
         args.pop('alwayson_scripts', None)
-
-        script_args = self.init_script_args(txt2imgreq, self.default_script_arg_txt2img, selectable_scripts, selectable_script_idx, script_runner)
-
         send_images = args.pop('send_images', True)
         args.pop('save_images', None)
 
@@ -245,37 +221,32 @@ class Api:
             p.scripts = script_runner
             p.outpath_grids = opts.outdir_grids or opts.outdir_txt2img_grids
             p.outpath_samples = opts.outdir_samples or opts.outdir_txt2img_samples
-
             shared.state.begin()
+            script_args = self.init_script_args(p, txt2imgreq, self.default_script_arg_txt2img, script_runner)
             if selectable_scripts is not None:
-                p.script_args = script_args
-                processed = scripts.scripts_txt2img.run(p, *p.script_args) # Need to pass args as list here
+                processed = scripts.scripts_txt2img.run(p, *script_args) # Need to pass args as list here
             else:
                 p.script_args = tuple(script_args) # Need to pass args as tuple here
                 processed = process_images(p)
             shared.state.end()
 
         b64images = list(map(encode_pil_to_base64, processed.images)) if send_images else []
-
         return TextToImageResponse(images=b64images, parameters=vars(txt2imgreq), info=processed.js())
 
     def img2imgapi(self, img2imgreq: StableDiffusionImg2ImgProcessingAPI):
         init_images = img2imgreq.init_images
         if init_images is None:
             raise HTTPException(status_code=404, detail="Init image not found")
-
         mask = img2imgreq.mask
         if mask:
             mask = decode_base64_to_image(mask)
-
         script_runner = scripts.scripts_img2img
         if not script_runner.scripts:
             script_runner.initialize_scripts(True)
             ui.create_ui()
         if not self.default_script_arg_img2img:
             self.default_script_arg_img2img = self.init_default_script_args(script_runner)
-        selectable_scripts, selectable_script_idx = self.get_selectable_script(img2imgreq.script_name, script_runner)
-
+        selectable_scripts, _selectable_script_idx = self.get_selectable_script(img2imgreq.script_name, script_runner)
         populate = img2imgreq.copy(update={  # Override __init__ params
             "sampler_name": validate_sampler_name(img2imgreq.sampler_name or img2imgreq.sampler_index),
             "do_not_save_samples": not img2imgreq.save_images,
@@ -284,15 +255,11 @@ class Api:
         })
         if populate.sampler_name:
             populate.sampler_index = None  # prevent a warning later on
-
         args = vars(populate)
         args.pop('include_init_images', None)  # this is meant to be done by "exclude": True in model, but it's for a reason that I cannot determine.
         args.pop('script_name', None)
         args.pop('script_args', None)  # will refeed them to the pipeline directly after initializing them
         args.pop('alwayson_scripts', None)
-
-        script_args = self.init_script_args(img2imgreq, self.default_script_arg_img2img, selectable_scripts, selectable_script_idx, script_runner)
-
         send_images = args.pop('send_images', True)
         args.pop('save_images', None)
 
@@ -302,22 +269,19 @@ class Api:
             p.scripts = script_runner
             p.outpath_grids = opts.outdir_img2img_grids
             p.outpath_samples = opts.outdir_img2img_samples
-
             shared.state.begin()
+            script_args = self.init_script_args(p, img2imgreq, self.default_script_arg_txt2img, script_runner)
             if selectable_scripts is not None:
-                p.script_args = script_args
-                processed = scripts.scripts_img2img.run(p, *p.script_args) # Need to pass args as list here
+                processed = scripts.scripts_img2img.run(p, *script_args) # Need to pass args as list here
             else:
                 p.script_args = tuple(script_args) # Need to pass args as tuple here
                 processed = process_images(p)
             shared.state.end()
 
         b64images = list(map(encode_pil_to_base64, processed.images)) if send_images else []
-
         if not img2imgreq.include_init_images:
             img2imgreq.init_images = None
             img2imgreq.mask = None
-
         return ImageToImageResponse(images=b64images, parameters=vars(img2imgreq), info=processed.js())
 
     def extras_single_image_api(self, req: ExtrasSingleImageRequest):
@@ -429,12 +393,11 @@ class Api:
 
     def get_config(self):
         options = {}
-        for key in shared.opts.data.keys():
-            metadata = shared.opts.data_labels.get(key)
-            if metadata is not None:
-                options.update({key: shared.opts.data.get(key, shared.opts.data_labels.get(key).default)})
+        for k in shared.opts.data.keys():
+            if shared.opts.data_labels.get(k) is not None:
+                options.update({k: shared.opts.data.get(k, shared.opts.data_labels.get(k).default)})
             else:
-                options.update({key: shared.opts.data.get(key, None)})
+                options.update({k: shared.opts.data.get(k, None)})
 
         return options
 
@@ -512,20 +475,20 @@ class Api:
             filename = create_embedding(**args) # create empty embedding
             sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings() # reload embeddings so new one can be immediately used
             shared.state.end()
-            return CreateResponse(info = "create embedding filename: {filename}".format(filename = filename))
+            return CreateResponse(info = f"create embedding filename: {filename}")
         except AssertionError as e:
             shared.state.end()
-            return TrainResponse(info = "create embedding error: {error}".format(error = e))
+            return TrainResponse(info = f"create embedding error: {e}")
 
     def create_hypernetwork(self, args: dict):
         try:
             shared.state.begin()
             filename = create_hypernetwork(**args) # create empty embedding # pylint: disable=E1111
             shared.state.end()
-            return CreateResponse(info = "create hypernetwork filename: {filename}".format(filename = filename))
+            return CreateResponse(info = f"create hypernetwork filename: {filename}")
         except AssertionError as e:
             shared.state.end()
-            return TrainResponse(info = "create hypernetwork error: {error}".format(error = e))
+            return TrainResponse(info = f"create hypernetwork error: {e}")
 
     def preprocess(self, args: dict):
         try:
@@ -535,13 +498,13 @@ class Api:
             return PreprocessResponse(info = 'preprocess complete')
         except KeyError as e:
             shared.state.end()
-            return PreprocessResponse(info = "preprocess error: invalid token: {error}".format(error = e))
+            return PreprocessResponse(info = f"preprocess error: invalid token: {e}")
         except AssertionError as e:
             shared.state.end()
-            return PreprocessResponse(info = "preprocess error: {error}".format(error = e))
+            return PreprocessResponse(info = f"preprocess error: {e}")
         except FileNotFoundError as e:
             shared.state.end()
-            return PreprocessResponse(info = 'preprocess error: {error}'.format(error = e))
+            return PreprocessResponse(info = f'preprocess error: {e}')
 
     def train_embedding(self, args: dict):
         try:
@@ -552,17 +515,17 @@ class Api:
             if not apply_optimizations:
                 sd_hijack.undo_optimizations()
             try:
-                embedding, filename = train_embedding(**args) # can take a long time to complete
+                _embedding, filename = train_embedding(**args) # can take a long time to complete
             except Exception as e:
                 error = e
             finally:
                 if not apply_optimizations:
                     sd_hijack.apply_optimizations()
                 shared.state.end()
-            return TrainResponse(info = "train embedding complete: filename: {filename} error: {error}".format(filename = filename, error = error))
+            return TrainResponse(info = f"train embedding complete: filename: {filename} error: {error}")
         except AssertionError as msg:
             shared.state.end()
-            return TrainResponse(info = "train embedding error: {msg}".format(msg = msg))
+            return TrainResponse(info = f"train embedding error: {msg}")
 
     def train_hypernetwork(self, args: dict):
         try:
@@ -574,7 +537,7 @@ class Api:
             if not apply_optimizations:
                 sd_hijack.undo_optimizations()
             try:
-                hypernetwork, filename = train_hypernetwork(**args)
+                _hypernetwork, filename = train_hypernetwork(**args)
             except Exception as e:
                 error = e
             finally:
@@ -583,10 +546,10 @@ class Api:
                 if not apply_optimizations:
                     sd_hijack.apply_optimizations()
                 shared.state.end()
-            return TrainResponse(info="train embedding complete: filename: {filename} error: {error}".format(filename=filename, error=error))
-        except AssertionError as msg:
+            return TrainResponse(info=f"train embedding complete: filename: {filename} error: {error}")
+        except AssertionError:
             shared.state.end()
-            return TrainResponse(info="train embedding error: {error}".format(error=error))
+            return TrainResponse(info=f"train embedding error: {error}")
 
     def shutdown(self):
         print('shutdown request received')
@@ -600,7 +563,8 @@ class Api:
 
     def get_memory(self):
         try:
-            import os, psutil
+            import os
+            import psutil
             process = psutil.Process(os.getpid())
             res = process.memory_info() # only rss is cross-platform guaranteed so we dont rely on other values
             ram_total = 100 * res.rss / process.memory_percent() # and total memory is calculated as actual value is not cross-platform safe
diff --git a/modules/api/models.py b/modules/api/models.py
index 4a70f440c..21d2c2663 100644
--- a/modules/api/models.py
+++ b/modules/api/models.py
@@ -1,11 +1,10 @@
 import inspect
-from pydantic import BaseModel, Field, create_model
-from typing import Any, Optional
+from typing import Any, Optional, Dict, List
+from pydantic import BaseModel, Field, create_model # pylint: disable=no-name-in-module
 from typing_extensions import Literal
 from inflection import underscore
 from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img
 from modules.shared import sd_upscalers, opts, parser
-from typing import Dict, List
 
 API_NOT_ALLOWED = [
     "self",
@@ -14,8 +13,6 @@ API_NOT_ALLOWED = [
     "outpath_samples",
     "outpath_grids",
     "sampler_index",
-    # "do_not_save_samples",
-    # "do_not_save_grid",
     "extra_generation_params",
     "overlay_images",
     "do_not_reload_embeddings",
@@ -48,7 +45,7 @@ class PydanticModelGenerator:
         class_instance = None,
         additional_fields = None,
     ):
-        def field_type_generator(k, v):
+        def field_type_generator(_k, v):
             # field_type = str if not overrides.get(k) else overrides[k]["type"]
             # print(k, v.annotation, v.default)
             field_type = v.annotation
@@ -76,23 +73,21 @@ class PydanticModelGenerator:
             for (k,v) in self._class_data.items() if k not in API_NOT_ALLOWED
         ]
 
-        for fields in additional_fields:
+        for fld in additional_fields:
             self._model_def.append(ModelDef(
-                field=underscore(fields["key"]),
-                field_alias=fields["key"],
-                field_type=fields["type"],
-                field_value=fields["default"],
-                field_exclude=fields["exclude"] if "exclude" in fields else False))
+                field=underscore(fld["key"]),
+                field_alias=fld["key"],
+                field_type=fld["type"],
+                field_value=fld["default"],
+                field_exclude=fld["exclude"] if "exclude" in fld else False))
 
     def generate_model(self):
         """
         Creates a pydantic BaseModel
         from the json and overrides provided at initialization
         """
-        fields = {
-            d.field: (d.field_type, Field(default=d.field_value, alias=d.field_alias, exclude=d.field_exclude)) for d in self._model_def
-        }
-        DynamicModel = create_model(self._model_name, **fields)
+        model_fields = { d.field: (d.field_type, Field(default=d.field_value, alias=d.field_alias, exclude=d.field_exclude)) for d in self._model_def }
+        DynamicModel = create_model(self._model_name, **model_fields)
         DynamicModel.__config__.allow_population_by_field_name = True
         DynamicModel.__config__.allow_mutation = True
         return DynamicModel
@@ -209,7 +204,7 @@ for key, metadata in opts.data_labels.items():
     value = opts.data.get(key)
     optType = opts.typemap.get(type(metadata.default), type(value))
 
-    if (metadata is not None):
+    if metadata is not None:
         fields.update({key: (Optional[optType], Field(
             default=metadata.default ,description=metadata.label))})
     else:
@@ -220,10 +215,11 @@ OptionsModel = create_model("Options", **fields)
 flags = {}
 _options = vars(parser)['_option_string_actions']
 for key in _options:
-    if(_options[key].dest != 'help'):
+    if _options[key].dest != 'help':
         flag = _options[key]
         _type = str
-        if _options[key].default is not None: _type = type(_options[key].default)
+        if _options[key].default is not None:
+            _type = type(_options[key].default)
         flags.update({flag.dest: (_type,Field(default=flag.default, description=flag.help))})
 
 FlagsModel = create_model("Flags", **flags)
@@ -288,4 +284,4 @@ class MemoryResponse(BaseModel):
 
 class ScriptsList(BaseModel):
     txt2img: list = Field(default=None,title="Txt2img", description="Titles of scripts (txt2img)")
-    img2img: list = Field(default=None,title="Img2img", description="Titles of scripts (img2img)")
\ No newline at end of file
+    img2img: list = Field(default=None,title="Img2img", description="Titles of scripts (img2img)")
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index 692f6451f..a8f20ca58 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -2,7 +2,7 @@ import argparse
 import os
 from modules.paths_internal import data_path, sd_default_config, sd_model_file
 
-parser = argparse.ArgumentParser(description="Stable Diffusion", formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=55,indent_increment=2,width=200))
+parser = argparse.ArgumentParser(description="Stable Diffusion", conflict_handler='resolve', formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=55, indent_increment=2, width=200))
 
 parser.add_argument("-f", action='store_true', help=argparse.SUPPRESS)  # allows running as root; implemented outside of webui
 parser.add_argument("--ui-settings-file", type=str, help=argparse.SUPPRESS, default=os.path.join(data_path, 'config.json'))
diff --git a/modules/img2img.py b/modules/img2img.py
index 302ae61ec..4a3472d27 100644
--- a/modules/img2img.py
+++ b/modules/img2img.py
@@ -151,7 +151,7 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s
     )
 
     p.scripts = modules.scripts.scripts_img2img
-    p.script_args = args
+    # p.script_args = args
 
     if mask:
         p.extra_generation_params["Mask blur"] = mask_blur
diff --git a/modules/processing.py b/modules/processing.py
index 1c6c36d32..ce2d9f86e 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -161,6 +161,7 @@ class StableDiffusionProcessing:
             self.seed_resize_from_w = 0
         self.scripts = None
         self.script_args = script_args
+        self.per_script_args = {}
         self.all_prompts = None
         self.all_negative_prompts = None
         self.all_seeds = None
diff --git a/modules/scripts.py b/modules/scripts.py
index 55418dc5b..f08adbb64 100644
--- a/modules/scripts.py
+++ b/modules/scripts.py
@@ -345,56 +345,56 @@ class ScriptRunner:
         script = self.selectable_scripts[script_index-1]
         if script is None:
             return None
-        script_args = args[script.args_from:script.args_to]
-        processed = script.run(p, *script_args)
+        args = p.per_script_args.get(script.title(), p.script_args[script.args_from:script.args_to])
+        processed = script.run(p, *args)
         shared.total_tqdm.clear()
         return processed
 
     def process(self, p, **kwargs):
         for script in self.alwayson_scripts:
             try:
-                script_args = p.script_args[script.args_from:script.args_to]
-                script.process(p, *script_args, **kwargs)
+                args = p.per_script_args.get(script.title(), p.script_args[script.args_from:script.args_to])
+                script.process(p, *args, **kwargs)
             except Exception as e:
                 errors.display(e, f'Running script process: {script.filename}')
 
     def before_process_batch(self, p, **kwargs):
         for script in self.alwayson_scripts:
             try:
-                script_args = p.script_args[script.args_from:script.args_to]
-                script.before_process_batch(p, *script_args, **kwargs)
+                args = p.per_script_args.get(script.title(), p.script_args[script.args_from:script.args_to])
+                script.before_process_batch(p, *args, **kwargs)
             except Exception as e:
                 errors.display(e, f'Running script before process batch: {script.filename}')
 
     def process_batch(self, p, **kwargs):
         for script in self.alwayson_scripts:
             try:
-                script_args = p.script_args[script.args_from:script.args_to]
-                script.process_batch(p, *script_args, **kwargs)
+                args = p.per_script_args.get(script.title(), p.script_args[script.args_from:script.args_to])
+                script.process_batch(p, *args, **kwargs)
             except Exception as e:
                 errors.display(e, f'Running script process batch: {script.filename}')
 
     def postprocess(self, p, processed):
         for script in self.alwayson_scripts:
             try:
-                script_args = p.script_args[script.args_from:script.args_to]
-                script.postprocess(p, processed, *script_args)
+                args = p.per_script_args.get(script.title(), p.script_args[script.args_from:script.args_to])
+                script.postprocess(p, processed, *args)
             except Exception as e:
                 errors.display(e, f'Running script postprocess: {script.filename}')
 
     def postprocess_batch(self, p, images, **kwargs):
         for script in self.alwayson_scripts:
             try:
-                script_args = p.script_args[script.args_from:script.args_to]
-                script.postprocess_batch(p, *script_args, images=images, **kwargs)
+                args = p.per_script_args.get(script.title(), p.script_args[script.args_from:script.args_to])
+                script.postprocess_batch(p, *args, images=images, **kwargs)
             except Exception as e:
                 errors.display(e, f'Running script before postprocess batch: {script.filename}')
 
     def postprocess_image(self, p, pp: PostprocessImageArgs):
         for script in self.alwayson_scripts:
             try:
-                script_args = p.script_args[script.args_from:script.args_to]
-                script.postprocess_image(p, pp, *script_args)
+                args = p.per_script_args.get(script.title(), p.script_args[script.args_from:script.args_to])
+                script.postprocess_image(p, pp, *args)
             except Exception as e:
                 errors.display(e, f'Running script postprocess image: {script.filename}')
 
diff --git a/modules/txt2img.py b/modules/txt2img.py
index 2fcb4c49d..dffe6d117 100644
--- a/modules/txt2img.py
+++ b/modules/txt2img.py
@@ -41,7 +41,7 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step
         override_settings=override_settings,
     )
     p.scripts = modules.scripts.scripts_txt2img
-    p.script_args = args
+    # p.script_args = args
     processed = modules.scripts.scripts_txt2img.run(p, *args)
     if processed is None:
         processed = process_images(p)
diff --git a/setup.py b/setup.py
index 3963832dd..804ee35b0 100644
--- a/setup.py
+++ b/setup.py
@@ -10,7 +10,7 @@ try:
     from modules.cmd_args import parser
 except:
     import argparse
-    parser = argparse.ArgumentParser(description="Stable Diffusion", formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=55,indent_increment=2,width=200))
+    parser = argparse.ArgumentParser(description="Stable Diffusion", conflict_handler='resolve', formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=55, indent_increment=2, width=200))
 
 
 class Dot(dict): # dot notation access to dictionary attributes

From 408147d9c4bf32e665a88b457ec27de746297821 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 13:25:20 -0400
Subject: [PATCH 29/69] critical fix

---
 extensions-builtin/sd-webui-controlnet | 2 +-
 javascript/extraNetworks.js            | 9 +++------
 modules/img2img.py                     | 2 +-
 modules/processing.py                  | 2 +-
 modules/scripts.py                     | 4 ++--
 modules/txt2img.py                     | 2 +-
 modules/ui_extra_networks.py           | 2 +-
 7 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index d520e9da0..4d4b1f8c0 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit d520e9da0014073162817ceb1eae241bccba67b9
+Subproject commit 4d4b1f8c00a0355d1517465ac3c0e801d5a2d194
diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js
index 19b69ec8b..3ecb34639 100644
--- a/javascript/extraNetworks.js
+++ b/javascript/extraNetworks.js
@@ -118,16 +118,16 @@ function readCardDescription(event, tabname, filename, descript){
     var textarea = gradioApp().querySelector("#" + tabname + '_description_filename  > label > textarea')
     var description_textarea = gradioApp().querySelector("#" + tabname+ '_description_input > label > textarea')
     var button = gradioApp().getElementById(tabname + '_read_description')
-
     textarea.value = filename
     description_textarea.value = descript
-
     updateInput(textarea)
     updateInput(description_textarea)
     button.click()
-
     event.stopPropagation()
     event.preventDefault()
+    requestGet("./sd_extra_networks/metadata", {"page": extraPage, "item": cardName}, (data) => {
+        if (data && data.metadata) extraNetworksShowMetadata(data.metadata)
+    }, () => {})
 }
 
 function extraNetworksSearchButton(tabs_id, event){
@@ -171,7 +171,6 @@ function extraNetworksShowMetadata(text){
     elem = document.createElement('pre')
     elem.classList.add('popup-metadata');
     elem.textContent = text;
-
     popup(elem);
 }
 
@@ -201,7 +200,6 @@ function requestGet(url, data, handler, errorHandler){
 
 function extraNetworksRequestMetadata(event, extraPage, cardName){
     showError = function(){ extraNetworksShowMetadata("there was an error getting metadata"); }
-
     requestGet("./sd_extra_networks/metadata", {"page": extraPage, "item": cardName}, function(data){
         if(data && data.metadata){
             extraNetworksShowMetadata(data.metadata)
@@ -209,6 +207,5 @@ function extraNetworksRequestMetadata(event, extraPage, cardName){
             showError()
         }
     }, showError)
-
     event.stopPropagation()
 }
diff --git a/modules/img2img.py b/modules/img2img.py
index 4a3472d27..302ae61ec 100644
--- a/modules/img2img.py
+++ b/modules/img2img.py
@@ -151,7 +151,7 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s
     )
 
     p.scripts = modules.scripts.scripts_img2img
-    # p.script_args = args
+    p.script_args = args
 
     if mask:
         p.extra_generation_params["Mask blur"] = mask_blur
diff --git a/modules/processing.py b/modules/processing.py
index ce2d9f86e..118996cab 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -160,7 +160,7 @@ class StableDiffusionProcessing:
             self.seed_resize_from_h = 0
             self.seed_resize_from_w = 0
         self.scripts = None
-        self.script_args = script_args
+        self.script_args = script_args or []
         self.per_script_args = {}
         self.all_prompts = None
         self.all_negative_prompts = None
diff --git a/modules/scripts.py b/modules/scripts.py
index f08adbb64..48df6c4dd 100644
--- a/modules/scripts.py
+++ b/modules/scripts.py
@@ -345,8 +345,8 @@ class ScriptRunner:
         script = self.selectable_scripts[script_index-1]
         if script is None:
             return None
-        args = p.per_script_args.get(script.title(), p.script_args[script.args_from:script.args_to])
-        processed = script.run(p, *args)
+        parsed = p.per_script_args.get(script.title(), args[script.args_from:script.args_to])
+        processed = script.run(p, *parsed)
         shared.total_tqdm.clear()
         return processed
 
diff --git a/modules/txt2img.py b/modules/txt2img.py
index dffe6d117..2fcb4c49d 100644
--- a/modules/txt2img.py
+++ b/modules/txt2img.py
@@ -41,7 +41,7 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step
         override_settings=override_settings,
     )
     p.scripts = modules.scripts.scripts_txt2img
-    # p.script_args = args
+    p.script_args = args
     processed = modules.scripts.scripts_txt2img.run(p, *args)
     if processed is None:
         processed = process_images(p)
diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py
index a8e3851b1..749a7789f 100644
--- a/modules/ui_extra_networks.py
+++ b/modules/ui_extra_networks.py
@@ -142,7 +142,7 @@ class ExtraNetworksPage:
             "card_clicked": onclick,
             "save_card_description": '"' + html.escape(f"""return saveCardDescription(event, {json.dumps(tabname)}, {json.dumps(item["local_preview"])})""") + '"',
             "save_card_preview": '"' + html.escape(f"""return saveCardPreview(event, {json.dumps(tabname)}, {json.dumps(item["local_preview"])})""") + '"',
-            "read_card_description": '"' + html.escape(f"""return readCardDescription(event, {json.dumps(tabname)}, {json.dumps(item["local_preview"])}, {json.dumps(item["description"])})""") + '"',
+            "read_card_description": '"' + html.escape(f"""return readCardDescription(event, {json.dumps(tabname)}, {json.dumps(item["local_preview"])}, {json.dumps(item["description"])}, {json.dumps(self.name)}, {json.dumps(item["name"])})""") + '"',
             "search_term": item.get("search_term", ""),
             "metadata_button": metadata_button,
         }

From 69ee51f7f2626ee5548064765ccae0df731826b7 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 14:16:39 -0400
Subject: [PATCH 30/69] fix extra networks

---
 ...ard.html => extra-networks-card-long.html} |  8 +-
 html/extra-networks-card-short.html           | 13 ++++
 javascript/extraNetworks.js                   | 73 ++++---------------
 modules/ui_extra_networks.py                  | 14 ++--
 modules/ui_extra_networks_checkpoints.py      |  2 +-
 5 files changed, 40 insertions(+), 70 deletions(-)
 rename html/{extra-networks-card.html => extra-networks-card-long.html} (52%)
 create mode 100644 html/extra-networks-card-short.html

diff --git a/html/extra-networks-card.html b/html/extra-networks-card-long.html
similarity index 52%
rename from html/extra-networks-card.html
rename to html/extra-networks-card-long.html
index 3cf6e2836..e2e01a15c 100644
--- a/html/extra-networks-card.html
+++ b/html/extra-networks-card-long.html
@@ -1,12 +1,10 @@
 <div class='card' style={style} onclick={card_clicked}>
-	{metadata_button}
-
 	<div class='actions'>
 		<div class='additional'>
 			<ul>
-				<li><a href="#" title="replace preview image with current selection" onclick={save_card_preview}>Replace preview</a></li>
-				<li><a href="#" title="replace preview description with current selection" onclick={save_card_description}>Replace description</a></li>
-				<li><a href="#" title="read description" onclick={read_card_description}>Read description</a></li>
+				<li><a href="#" title="set preview image with current selection" onclick={save_card_preview}>Replace preview</a></li>
+				<li><a href="#" title="read description" onclick={read_card_description}>Description</a> <a href="#" title="[replace]" onclick={save_card_description}>[Replace]</a></li>
+				<li><a href="#" title="read metadata" onclick={read_card_metadata}>Metadata</a></li>
 			</ul>
 			<span style="display:none" class='search_term'>{search_term}</span>
 		</div>
diff --git a/html/extra-networks-card-short.html b/html/extra-networks-card-short.html
new file mode 100644
index 000000000..3dc9e46a2
--- /dev/null
+++ b/html/extra-networks-card-short.html
@@ -0,0 +1,13 @@
+<div class='card' style={style} onclick={card_clicked}>
+	<div class='actions'>
+		<div class='additional'>
+			<ul>
+				<li><a href="#" title="set preview image with current selection" onclick={save_card_preview}>Replace preview</a></li>
+				<li><a href="#" title="read description" onclick={read_card_description}>Description</a> <a href="#" title="[replace]" onclick={save_card_description}>[Replace]</a></li>
+			</ul>
+			<span style="display:none" class='search_term'>{search_term}</span>
+		</div>
+		<span class='name'>{name}</span>
+		<span class='description'>{description}</span>
+	</div>
+</div>
diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js
index 3ecb34639..bff379beb 100644
--- a/javascript/extraNetworks.js
+++ b/javascript/extraNetworks.js
@@ -1,22 +1,17 @@
-
 function setupExtraNetworksForTab(tabname){
     gradioApp().querySelector('#'+tabname+'_extra_tabs').classList.add('extra-networks')
-
     var tabs = gradioApp().querySelector('#'+tabname+'_extra_tabs > div')
     var search = gradioApp().querySelector('#'+tabname+'_extra_search textarea')
     var refresh = gradioApp().getElementById(tabname+'_extra_refresh')
     var descriptInput = gradioApp().getElementById(tabname+ '_description_input')
     var close = gradioApp().getElementById(tabname+'_extra_close')
-
     search.classList.add('search')
     tabs.appendChild(search)
     tabs.appendChild(refresh)
     tabs.appendChild(close)
     tabs.appendChild(descriptInput)
-
     search.addEventListener("input", function(evt){
         searchTerm = search.value.toLowerCase()
-
         gradioApp().querySelectorAll('#'+tabname+'_extra_tabs div.card').forEach(function(elem){
             text = elem.querySelector('.name').textContent.toLowerCase() + " " + elem.querySelector('.search_term').textContent.toLowerCase()
             elem.style.display = text.indexOf(searchTerm) == -1 ? "none" : ""
@@ -29,19 +24,13 @@ var activePromptTextarea = {};
 function setupExtraNetworks(){
     setupExtraNetworksForTab('txt2img')
     setupExtraNetworksForTab('img2img')
-
     function registerPrompt(tabname, id){
         var textarea = gradioApp().querySelector("#" + id + " > label > textarea");
-
-        if (! activePromptTextarea[tabname]){
-            activePromptTextarea[tabname] = textarea
-        }
-
-		textarea.addEventListener("focus", function(){
+        if ( !activePromptTextarea[tabname]) activePromptTextarea[tabname] = textarea
+        textarea.addEventListener("focus", function(){
             activePromptTextarea[tabname] = textarea;
-		});
+        });
     }
-
     registerPrompt('txt2img', 'txt2img_prompt')
     registerPrompt('txt2img', 'txt2img_neg_prompt')
     registerPrompt('img2img', 'img2img_prompt')
@@ -49,14 +38,12 @@ function setupExtraNetworks(){
 }
 
 onUiLoaded(setupExtraNetworks)
-
 var re_extranet   =    /<([^:]+:[^:]+):[\d\.]+>/;
 var re_extranet_g = /\s+<([^:]+:[^:]+):[\d\.]+>/g;
 
 function tryToRemoveExtraNetworkFromPrompt(textarea, text){
     var m = text.match(re_extranet)
     if(! m) return false
-
     var partToSearch = m[1]
     var replaced = false
     var newTextareaText = textarea.value.replaceAll(re_extranet_g, function(found, index){
@@ -67,34 +54,25 @@ function tryToRemoveExtraNetworkFromPrompt(textarea, text){
         }
         return found;
     })
-
     if(replaced){
         textarea.value = newTextareaText
         return true;
     }
-
     return false
 }
 
 function cardClicked(tabname, textToAdd, allowNegativePrompt){
     var textarea = allowNegativePrompt ? activePromptTextarea[tabname] : gradioApp().querySelector("#" + tabname + "_prompt > label > textarea")
-
-    if(! tryToRemoveExtraNetworkFromPrompt(textarea, textToAdd)){
-        textarea.value = textarea.value + opts.extra_networks_add_text_separator + textToAdd
-    }
-
+    if (!tryToRemoveExtraNetworkFromPrompt(textarea, textToAdd)) textarea.value = textarea.value + opts.extra_networks_add_text_separator + textToAdd
     updateInput(textarea)
 }
 
 function saveCardPreview(event, tabname, filename){
     var textarea = gradioApp().querySelector("#" + tabname + '_preview_filename  > label > textarea')
     var button = gradioApp().getElementById(tabname + '_save_preview')
-
     textarea.value = filename
     updateInput(textarea)
-
     button.click()
-
     event.stopPropagation()
     event.preventDefault()
 }
@@ -103,18 +81,15 @@ function saveCardDescription(event, tabname, filename, descript){
     var textarea = gradioApp().querySelector("#" + tabname + '_description_filename  > label > textarea')
     var button = gradioApp().getElementById(tabname + '_save_description')
     var description = gradioApp().getElementById(tabname+ '_description_input')
-
     textarea.value = filename
     description.value=descript
     updateInput(textarea)
-
     button.click()
-
     event.stopPropagation()
     event.preventDefault()
 }
 
-function readCardDescription(event, tabname, filename, descript){
+function readCardDescription(event, tabname, filename, descript, extraPage, cardName){
     var textarea = gradioApp().querySelector("#" + tabname + '_description_filename  > label > textarea')
     var description_textarea = gradioApp().querySelector("#" + tabname+ '_description_input > label > textarea')
     var button = gradioApp().getElementById(tabname + '_read_description')
@@ -125,16 +100,12 @@ function readCardDescription(event, tabname, filename, descript){
     button.click()
     event.stopPropagation()
     event.preventDefault()
-    requestGet("./sd_extra_networks/metadata", {"page": extraPage, "item": cardName}, (data) => {
-        if (data && data.metadata) extraNetworksShowMetadata(data.metadata)
-    }, () => {})
 }
 
 function extraNetworksSearchButton(tabs_id, event){
     searchTextarea = gradioApp().querySelector("#" + tabs_id + ' > div > textarea')
     button = event.target
     text = button.classList.contains("search-all") ? "" : button.textContent.trim()
-
     searchTextarea.value = text
     updateInput(searchTextarea)
 }
@@ -146,39 +117,39 @@ function popup(contents){
         globalPopup = document.createElement('div')
         globalPopup.onclick = function(){ globalPopup.style.display = "none"; };
         globalPopup.classList.add('global-popup');
-
         var close = document.createElement('div')
         close.classList.add('global-popup-close');
         close.onclick = function(){ globalPopup.style.display = "none"; };
         close.title = "Close";
         globalPopup.appendChild(close)
-
         globalPopupInner = document.createElement('div')
         globalPopupInner.onclick = function(event){ event.stopPropagation(); return false; };
         globalPopupInner.classList.add('global-popup-inner');
         globalPopup.appendChild(globalPopupInner)
-
         gradioApp().appendChild(globalPopup);
     }
-
     globalPopupInner.innerHTML = '';
     globalPopupInner.appendChild(contents);
-
     globalPopup.style.display = "flex";
 }
 
-function extraNetworksShowMetadata(text){
-    elem = document.createElement('pre')
-    elem.classList.add('popup-metadata');
-    elem.textContent = text;
-    popup(elem);
+function readCardMetadata(event, extraPage, cardName){
+    requestGet("./sd_extra_networks/metadata", {"page": extraPage, "item": cardName}, function(data){
+        if (data && data.metadata){
+            elem = document.createElement('pre')
+            elem.classList.add('popup-metadata');
+            elem.textContent = data.metadata;
+            popup(elem);
+        }
+    }, () => {})
+    event.stopPropagation()
+    event.preventDefault()
 }
 
 function requestGet(url, data, handler, errorHandler){
     var xhr = new XMLHttpRequest();
     var args = Object.keys(data).map(function(k){ return encodeURIComponent(k) + '=' + encodeURIComponent(data[k]) }).join('&')
     xhr.open("GET", url + "?" + args, true);
-
     xhr.onreadystatechange = function () {
         if (xhr.readyState === 4) {
             if (xhr.status === 200) {
@@ -197,15 +168,3 @@ function requestGet(url, data, handler, errorHandler){
     var js = JSON.stringify(data);
     xhr.send(js);
 }
-
-function extraNetworksRequestMetadata(event, extraPage, cardName){
-    showError = function(){ extraNetworksShowMetadata("there was an error getting metadata"); }
-    requestGet("./sd_extra_networks/metadata", {"page": extraPage, "item": cardName}, function(data){
-        if(data && data.metadata){
-            extraNetworksShowMetadata(data.metadata)
-        } else{
-            showError()
-        }
-    }, showError)
-    event.stopPropagation()
-}
diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py
index 749a7789f..acaf4df64 100644
--- a/modules/ui_extra_networks.py
+++ b/modules/ui_extra_networks.py
@@ -54,7 +54,8 @@ class ExtraNetworksPage:
     def __init__(self, title):
         self.title = title
         self.name = title.lower()
-        self.card_page = shared.html("extra-networks-card.html")
+        self.card_long = shared.html("extra-networks-card-long.html")
+        self.card_short = shared.html("extra-networks-card-short.html")
         self.allow_negative_prompt = False
         self.metadata = {}
 
@@ -128,10 +129,6 @@ class ExtraNetworksPage:
         height = f"height: {shared.opts.extra_networks_card_height}px;" if shared.opts.extra_networks_card_height else ''
         width = f"width: {shared.opts.extra_networks_card_width}px;" if shared.opts.extra_networks_card_width else ''
         background_image = f"background-image: url(\"{html.escape(preview)}\");" if preview else ''
-        metadata_button = ""
-        metadata = item.get("metadata")
-        if metadata:
-            metadata_button = f"<div class='metadata-button' title='Show metadata' onclick='extraNetworksRequestMetadata(event, {json.dumps(self.name)}, {json.dumps(item['name'])})'></div>"
         args = {
             "style": f"'{height}{width}{background_image}'",
             "prompt": item.get("prompt", None),
@@ -144,9 +141,12 @@ class ExtraNetworksPage:
             "save_card_preview": '"' + html.escape(f"""return saveCardPreview(event, {json.dumps(tabname)}, {json.dumps(item["local_preview"])})""") + '"',
             "read_card_description": '"' + html.escape(f"""return readCardDescription(event, {json.dumps(tabname)}, {json.dumps(item["local_preview"])}, {json.dumps(item["description"])}, {json.dumps(self.name)}, {json.dumps(item["name"])})""") + '"',
             "search_term": item.get("search_term", ""),
-            "metadata_button": metadata_button,
+            "read_card_metadata": '"' + html.escape(f"""return readCardMetadata(event, {json.dumps(self.name)}, {json.dumps(item["name"])})""") + '"',
         }
-        return self.card_page.format(**args)
+        if item.get("metadata"):
+            return self.card_long.format(**args)
+        else:
+            return self.card_short.format(**args)
 
     def find_preview(self, path):
         """
diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py
index 2ddf9b70d..fe65d99df 100644
--- a/modules/ui_extra_networks_checkpoints.py
+++ b/modules/ui_extra_networks_checkpoints.py
@@ -15,7 +15,7 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage):
     def list_items(self):
         checkpoint: sd_models.CheckpointInfo
         for name, checkpoint in sd_models.checkpoints_list.items():
-            path, ext = os.path.splitext(checkpoint.filename)
+            path, _ext = os.path.splitext(checkpoint.filename)
             yield {
                 "name": checkpoint.name_for_extra,
                 "filename": path,

From 3c410561232abc58d0c0306757668d3dc178f2c0 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 16:04:10 -0400
Subject: [PATCH 31/69] make clip skip persisent

---
 modules/ui.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/ui.py b/modules/ui.py
index c811cd3eb..12e888308 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -354,8 +354,8 @@ def create_ui():
                                     batch_size = gr.Slider(minimum=1, maximum=32, step=1, label='Batch size', value=1, elem_id="txt2img_batch_size")
                     elif category == "cfg":
                         with FormRow():
-                            cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.5, label='CFG Scale', value=7.0, elem_id="txt2img_cfg_scale")
-                            clip_skip = gr.Slider(label='CLIP Skip', value=1, minimum=1, maximum=4, step=1, elem_id='txt2img_clip_skip', interactive=True)
+                            cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.5, label='CFG Scale', value=6.0, elem_id="txt2img_cfg_scale")
+                            clip_skip = gr.Slider(label='CLIP Skip', value=shared.opts.CLIP_stop_at_last_layers, minimum=1, maximum=4, step=1, elem_id='txt2img_clip_skip', interactive=True)
                             clip_skip.change(fn=change_clip_skip, show_progress=False, inputs=clip_skip)
                     elif category == "seed":
                         seed, reuse_seed, subseed, reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, seed_checkbox = create_seed_inputs('txt2img')

From 2220316920cefc3d29d812aaf901d9714a42ec7d Mon Sep 17 00:00:00 2001
From: David Pina <pinvalda@gmail.com>
Date: Sat, 29 Apr 2023 22:41:24 +0200
Subject: [PATCH 32/69] Fix typo that prevents training Textual Inversion

There was a small typo in line 529.

shared.ops.embeddings_train_log caused an attribute not found exception when training TIs.
---
 modules/textual_inversion/textual_inversion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index 75bd1cd88..2180d7f32 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -526,7 +526,7 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
                     save_embedding(embedding, optimizer, checkpoint, embedding_name_every, last_saved_file, remove_cached_checksum=True)
                     embedding_yet_to_be_embedded = True
 
-                write_loss(log_directory, shared.ops.embeddings_train_log, embedding.step, steps_per_epoch, {
+                write_loss(log_directory, shared.opts.embeddings_train_log, embedding.step, steps_per_epoch, {
                     "loss": f"{loss_step:.7f}",
                     "learn_rate": scheduler.learn_rate
                 })

From ba50dbbdbf946d3b724c84e5c711caddfc8c4770 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 17:35:52 -0400
Subject: [PATCH 33/69] add memory monitor

---
 cli/modules/bench.py                        |  2 +-
 extensions-builtin/sd-extension-system-info |  2 +-
 modules/cmd_args.py                         | 12 +++++
 modules/img2img.py                          | 52 +++++----------------
 modules/memmon.py                           | 13 +-----
 modules/processing.py                       | 31 ++++++++++--
 modules/scripts.py                          |  3 +-
 modules/sd_samplers_compvis.py              |  5 --
 modules/sd_samplers_kdiffusion.py           |  1 -
 modules/shared.py                           | 14 +-----
 modules/txt2img.py                          | 12 ++---
 scripts/postprocessing_upscale.py           | 16 +++----
 scripts/xyz_grid.py                         |  1 -
 webui.py                                    |  2 +-
 14 files changed, 70 insertions(+), 96 deletions(-)

diff --git a/cli/modules/bench.py b/cli/modules/bench.py
index 801c9ccfd..094b73f63 100755
--- a/cli/modules/bench.py
+++ b/cli/modules/bench.py
@@ -8,7 +8,7 @@ import io
 import json
 import time
 from PIL import Image
-import sdapi as sdapi
+import sdapi
 from util import Map, log
 
 
diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info
index cc86ce888..70ab5cf31 160000
--- a/extensions-builtin/sd-extension-system-info
+++ b/extensions-builtin/sd-extension-system-info
@@ -1 +1 @@
-Subproject commit cc86ce8887f041e88e67d336044190fd0296fd74
+Subproject commit 70ab5cf312be0fa913c5ba6ab85fbb64430507e2
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index a8f20ca58..3bca9b357 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -79,6 +79,18 @@ def compatibility_args(opts, args):
     opts.use_old_karras_scheduler_sigmas = False
     opts.no_dpmpp_sde_batch_determinism = False
     opts.lora_apply_to_outputs = False
+    opts.do_not_show_images = False
+    opts.add_model_hash_to_info = True
+    opts.add_model_name_to_info = True
+    opts.js_modal_lightbox = True
+    opts.js_modal_lightbox_initially_zoomed = True
+    opts.show_progress_in_title = False
+    opts.sd_vae_as_default = True
+    opts.enable_emphasis = True
+    opts.enable_batch_seeds = True
+    opts.multiple_tqdm = False
+    opts.print_hypernet_extra = False
+    opts.dimensions_and_batch_together = True
 
     parser.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir)
     args = parser.parse_args()
diff --git a/modules/img2img.py b/modules/img2img.py
index 302ae61ec..de889d4b7 100644
--- a/modules/img2img.py
+++ b/modules/img2img.py
@@ -1,47 +1,35 @@
 import os
-
 import numpy as np
 from PIL import Image, ImageOps, ImageFilter, ImageEnhance, ImageChops, UnidentifiedImageError
-
+import modules.scripts
 from modules import sd_samplers
 from modules.generation_parameters_copypaste import create_override_settings_dict
-from modules.processing import Processed, StableDiffusionProcessingImg2Img, process_images
-from modules.shared import opts, state
-import modules.shared as shared
-import modules.processing as processing
+from modules.processing import Processed, StableDiffusionProcessingImg2Img, process_images, memory_stats
+from modules.shared import opts, cmd_opts, log, state, listfiles, sd_model
 from modules.ui import plaintext_to_html
-import modules.scripts
+import modules.processing as processing
 
 
 def process_batch(p, input_dir, output_dir, inpaint_mask_dir, args):
     processing.fix_seed(p)
-
-    images = shared.listfiles(input_dir)
-
+    images = listfiles(input_dir)
     is_inpaint_batch = False
     if inpaint_mask_dir:
-        inpaint_masks = shared.listfiles(inpaint_mask_dir)
+        inpaint_masks = listfiles(inpaint_mask_dir)
         is_inpaint_batch = len(inpaint_masks) > 0
     if is_inpaint_batch:
         print(f"\nInpaint batch is enabled. {len(inpaint_masks)} masks found.")
-
     print(f"Will process {len(images)} images, creating {p.n_iter * p.batch_size} new images for each.")
-
     save_normally = output_dir == ''
-
     p.do_not_save_grid = True
     p.do_not_save_samples = not save_normally
-
     state.job_count = len(images) * p.n_iter
-
     for i, image in enumerate(images):
         state.job = f"{i+1} out of {len(images)}"
         if state.skipped:
             state.skipped = False
-
         if state.interrupted:
             break
-
         try:
             img = Image.open(image)
         except UnidentifiedImageError:
@@ -62,26 +50,24 @@ def process_batch(p, input_dir, output_dir, inpaint_mask_dir, args):
         proc = modules.scripts.scripts_img2img.run(p, *args)
         if proc is None:
             proc = process_images(p)
-
         for n, processed_image in enumerate(proc.images):
             filename = os.path.basename(image)
-
             if n > 0:
                 left, right = os.path.splitext(filename)
                 filename = f"{left}-{n}{right}"
-
             if not save_normally:
                 os.makedirs(output_dir, exist_ok=True)
                 if processed_image.mode == 'RGBA':
                     processed_image = processed_image.convert("RGB")
                 processed_image.save(os.path.join(output_dir, filename))
+    if cmd_opts.debug:
+        log.info(f'Processed: {len(images)} Memory: {memory_stats()} batch')
 
 
 def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_styles, init_img, sketch, init_img_with_mask, inpaint_color_sketch, inpaint_color_sketch_orig, init_img_inpaint, init_mask_inpaint, steps: int, sampler_index: int, mask_blur: int, mask_alpha: float, inpainting_fill: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, denoising_strength: float, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, seed_enable_extras: bool, height: int, width: int, resize_mode: int, inpaint_full_res: bool, inpaint_full_res_padding: int, inpainting_mask_invert: int, img2img_batch_input_dir: str, img2img_batch_output_dir: str, img2img_batch_inpaint_mask_dir: str, override_settings_texts, *args): # pylint: disable=unused-argument
     override_settings = create_override_settings_dict(override_settings_texts)
 
     is_batch = mode == 5
-
     if mode == 0:  # img2img
         image = init_img.convert("RGB")
         mask = None
@@ -108,15 +94,12 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s
     else:
         image = None
         mask = None
-
-    # Use the EXIF orientation of photos taken by smartphones.
     if image is not None:
         image = ImageOps.exif_transpose(image)
-
     assert 0. <= denoising_strength <= 1., 'can only work with strength in [0.0, 1.0]'
 
     p = StableDiffusionProcessingImg2Img(
-        sd_model=shared.sd_model,
+        sd_model=sd_model,
         outpath_samples=opts.outdir_samples or opts.outdir_img2img_samples,
         outpath_grids=opts.outdir_grids or opts.outdir_img2img_grids,
         prompt=prompt,
@@ -149,31 +132,20 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s
         inpainting_mask_invert=inpainting_mask_invert,
         override_settings=override_settings,
     )
-
     p.scripts = modules.scripts.scripts_img2img
     p.script_args = args
-
     if mask:
         p.extra_generation_params["Mask blur"] = mask_blur
-
     if is_batch:
-        assert not shared.cmd_opts.hide_ui_dir_config, "Launched with --hide-ui-dir-config, batch img2img disabled"
-
+        assert not cmd_opts.hide_ui_dir_config, "Launched with --hide-ui-dir-config, batch img2img disabled"
         process_batch(p, img2img_batch_input_dir, img2img_batch_output_dir, img2img_batch_inpaint_mask_dir, args)
-
         processed = Processed(p, [], p.seed, "")
     else:
         processed = modules.scripts.scripts_img2img.run(p, *args)
         if processed is None:
             processed = process_images(p)
-
     p.close()
-
-    shared.total_tqdm.clear()
-
     generation_info_js = processed.js()
-
-    if opts.do_not_show_images:
-        processed.images = []
-
+    if cmd_opts.debug:
+        log.info(f'Processed: {len(processed.images)} Memory: {memory_stats()} img')
     return processed.images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments)
diff --git a/modules/memmon.py b/modules/memmon.py
index 8c257b92f..9b013e6b4 100644
--- a/modules/memmon.py
+++ b/modules/memmon.py
@@ -1,7 +1,6 @@
 import threading
 import time
 from collections import defaultdict
-
 import torch
 
 
@@ -17,11 +16,9 @@ class MemUsageMonitor(threading.Thread):
         self.name = name
         self.device = device
         self.opts = opts
-
         self.daemon = True
         self.run_flag = threading.Event()
         self.data = defaultdict(int)
-
         if not torch.cuda.is_available():
             self.disabled = True
         else:
@@ -39,37 +36,29 @@ class MemUsageMonitor(threading.Thread):
     def run(self):
         if self.disabled:
             return
-
         while True:
             self.run_flag.wait()
-
             torch.cuda.reset_peak_memory_stats()
             self.data.clear()
-
             if self.opts.memmon_poll_rate <= 0:
                 self.run_flag.clear()
                 continue
-
             self.data["min_free"] = self.cuda_mem_get_info()[0]
-
             while self.run_flag.is_set():
-                free, total = self.cuda_mem_get_info()
+                free, _total = self.cuda_mem_get_info()
                 self.data["min_free"] = min(self.data["min_free"], free)
-
                 time.sleep(1 / self.opts.memmon_poll_rate)
 
     def dump_debug(self):
         print(self, 'recorded data:')
         for k, v in self.read().items():
             print(k, -(v // -(1024 ** 2)))
-
         print(self, 'raw torch memory stats:')
         tm = torch.cuda.memory_stats(self.device)
         for k, v in tm.items():
             if 'bytes' not in k:
                 continue
             print('\t' if 'peak' in k else '', k, -(v // -(1024 ** 2)))
-
         print(torch.cuda.memory_summary())
 
     def monitor(self):
diff --git a/modules/processing.py b/modules/processing.py
index 118996cab..e793f12a3 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -6,6 +6,7 @@ import random
 import logging
 from typing import Any, Dict, List
 
+import psutil
 import torch
 import numpy as np
 from PIL import Image, ImageFilter, ImageOps
@@ -41,6 +42,33 @@ opt_C = 4
 opt_f = 8
 
 
+def memory_stats():
+    def gb(val: float):
+        return round(val / 1024 / 1024 / 1024, 2)
+    mem = {}
+    try:
+        process = psutil.Process(os.getpid())
+        res = process.memory_info()
+        ram_total = 100 * res.rss / process.memory_percent()
+        ram = { 'used': gb(res.rss), 'total': gb(ram_total) }
+        mem.update({ 'ram': ram })
+    except Exception as e:
+        mem.update({ 'ram': e })
+    try:
+        if torch.cuda.is_available():
+            s = torch.cuda.mem_get_info()
+            gpu = { 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }
+            s = dict(torch.cuda.memory_stats(shared.device))
+            mem.update({
+                'gpu': gpu,
+                'retries': s['num_alloc_retries'],
+                'oom': s['num_ooms']
+            })
+    except:
+        pass
+    return mem
+
+
 def setup_color_correction(image):
     logging.info("Calibrating color correction.")
     correction_target = cv2.cvtColor(np.asarray(image.copy()), cv2.COLOR_RGB2LAB)
@@ -317,7 +345,6 @@ class Processed:
         self.seed = int(self.seed if type(self.seed) != list else self.seed[0]) if self.seed is not None else -1
         self.subseed = int(self.subseed if type(self.subseed) != list else self.subseed[0]) if self.subseed is not None else -1
         self.is_using_inpainting_conditioning = p.is_using_inpainting_conditioning
-
         self.all_prompts = all_prompts or p.all_prompts or [self.prompt]
         self.all_negative_prompts = all_negative_prompts or p.all_negative_prompts or [self.negative_prompt]
         self.all_seeds = all_seeds or p.all_seeds or [self.seed]
@@ -892,8 +919,6 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
             if not state.processing_has_refined_job_count:
                 if state.job_count == -1:
                     state.job_count = self.n_iter
-
-                shared.total_tqdm.updateTotal((self.steps + (self.hr_second_pass_steps or self.steps)) * state.job_count)
                 state.job_count = state.job_count * 2
                 state.processing_has_refined_job_count = True
 
diff --git a/modules/scripts.py b/modules/scripts.py
index 48df6c4dd..fa8a3cef8 100644
--- a/modules/scripts.py
+++ b/modules/scripts.py
@@ -3,7 +3,7 @@ import re
 import sys
 from collections import namedtuple
 import gradio as gr
-from modules import shared, paths, script_callbacks, extensions, script_loading, scripts_postprocessing, errors
+from modules import paths, script_callbacks, extensions, script_loading, scripts_postprocessing, errors
 
 AlwaysVisible = object()
 
@@ -347,7 +347,6 @@ class ScriptRunner:
             return None
         parsed = p.per_script_args.get(script.title(), args[script.args_from:script.args_to])
         processed = script.run(p, *parsed)
-        shared.total_tqdm.clear()
         return processed
 
     def process(self, p, **kwargs):
diff --git a/modules/sd_samplers_compvis.py b/modules/sd_samplers_compvis.py
index bfcc55749..8de719323 100644
--- a/modules/sd_samplers_compvis.py
+++ b/modules/sd_samplers_compvis.py
@@ -109,7 +109,6 @@ class VanillaStableDiffusionSampler:
             else:
                 cond = {"c_concat": [image_conditioning], "c_crossattn": [cond]}
                 unconditional_conditioning = {"c_concat": [image_conditioning], "c_crossattn": [unconditional_conditioning]}
-
         return x, ts, cond, unconditional_conditioning
 
     def update_step(self, last_latent):
@@ -117,17 +116,13 @@ class VanillaStableDiffusionSampler:
             self.last_latent = self.init_latent * self.mask + self.nmask * last_latent
         else:
             self.last_latent = last_latent
-
         sd_samplers_common.store_latent(self.last_latent)
-
         self.step += 1
         state.sampling_step = self.step
-        shared.total_tqdm.update()
 
     def after_sample(self, x, ts, cond, uncond, res):
         if not self.is_unipc:
             self.update_step(res[1])
-
         return x, ts, cond, uncond, res
 
     def unipc_after_update(self, x, model_x):
diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py
index 3e4f882c6..a30d351fc 100644
--- a/modules/sd_samplers_kdiffusion.py
+++ b/modules/sd_samplers_kdiffusion.py
@@ -231,7 +231,6 @@ class KDiffusionSampler:
             raise sd_samplers_common.InterruptedException
 
         state.sampling_step = step
-        shared.total_tqdm.update()
 
     def launch_sampling(self, steps, func):
         state.sampling_steps = steps
diff --git a/modules/shared.py b/modules/shared.py
index c18e31021..0dfefa8c4 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -223,15 +223,12 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), {
     "sd_checkpoint_cache": OptionInfo(0, "Model checkpoints to cache in RAM", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}),
     "sd_vae_checkpoint_cache": OptionInfo(0, "VAE checkpoints to cache in RAM", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}),
     "sd_vae": OptionInfo("Automatic", "Select VAE", gr.Dropdown, lambda: {"choices": shared_items.sd_vae_items()}, refresh=shared_items.refresh_vae_list),
-    "sd_vae_as_default": OptionInfo(True, "Ignore selected VAE for stable diffusion checkpoints that have their own .vae.pt next to them", gr.Checkbox, {"visible": False}),
     "inpainting_mask_weight": OptionInfo(1.0, "Inpainting conditioning mask strength", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),
     "initial_noise_multiplier": OptionInfo(1.0, "Noise multiplier for img2img", gr.Slider, {"minimum": 0.5, "maximum": 1.5, "step": 0.01}),
     "img2img_color_correction": OptionInfo(False, "Apply color correction to img2img results to match original colors."),
     "img2img_fix_steps": OptionInfo(False, "For image processing do exactly the amount of steps as specified."),
     "img2img_background_color": OptionInfo("#ffffff", "With img2img, fill image's transparent parts with this color.", ui_components.FormColorPicker, {}),
     "enable_quantization": OptionInfo(True, "Enable quantization in K samplers for sharper and cleaner results. This may change existing seeds."),
-    "enable_emphasis": OptionInfo(True, "Emphasis: use (text) to make model pay more attention to text and [text] to make it pay less attention", gr.Checkbox, {"visible": False}),
-    "enable_batch_seeds": OptionInfo(True, "Make K-diffusion samplers produce same images in a batch as when making a single image", gr.Checkbox, {"visible": False}),
     "comma_padding_backtrack": OptionInfo(20, "Increase coherency by padding from the last comma within n tokens when using more than 75 tokens", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }),
     "CLIP_stop_at_last_layers": OptionInfo(1, "Clip skip", gr.Slider, {"minimum": 1, "maximum": 12, "step": 1, "visible": False}),
     "upcast_attn": OptionInfo(False, "Upcast cross attention layer to float32"),
@@ -241,9 +238,6 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), {
     "sub_quad_kv_chunk_size": OptionInfo(512, "Sub-quadratic cross-attentionkv chunk size for the sub-quadratic cross-attention layer optimization to use", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8}),
     "sub_quad_chunk_threshold": OptionInfo(80, "Sub-quadratic cross-attention percentage of VRAM chunking threshold", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}),
     "always_batch_cond_uncond": OptionInfo(False, "Disables cond/uncond batching that is enabled to save memory with --medvram or --lowvram"),
-    "multiple_tqdm": OptionInfo(False, "Add a second progress bar to the console that shows progress for an entire job.", gr.Checkbox, {"visible": False}),
-    "print_hypernet_extra": OptionInfo(False, "Print extra hypernetwork information to console.", gr.Checkbox, {"visible": False}),
-    "dimensions_and_batch_together": OptionInfo(True, "", gr.Checkbox, {"visible": False}),
 }))
 
 options_templates.update(options_section(('system-paths', "System Paths"), {
@@ -387,16 +381,10 @@ options_templates.update(options_section(('ui', "User interface"), {
     "return_grid": OptionInfo(True, "Show grid in results for web"),
     "return_mask": OptionInfo(False, "For inpainting, include the greyscale mask in results for web"),
     "return_mask_composite": OptionInfo(False, "For inpainting, include masked composite in results for web"),
-    "do_not_show_images": OptionInfo(False, "Do not show any images in results for web"),
-    "add_model_hash_to_info": OptionInfo(True, "Add model hash to generation information"),
-    "add_model_name_to_info": OptionInfo(True, "Add model name to generation information"),
     "disable_weights_auto_swap": OptionInfo(True, "Do not change the selected model when reading generation parameters."),
     "send_seed": OptionInfo(True, "Send seed when sending prompt or image to other interface"),
     "send_size": OptionInfo(True, "Send size when sending prompt or image to another interface"),
     "font": OptionInfo("", "Font for image grids that have text"),
-    "js_modal_lightbox": OptionInfo(True, "Enable full page image viewer", gr.Checkbox, {"visible": False}),
-    "js_modal_lightbox_initially_zoomed": OptionInfo(True, "Show images zoomed in by default in full page image viewer", gr.Checkbox, {"visible": False}),
-    "show_progress_in_title": OptionInfo(False, "Show generation progress in window title.", gr.Checkbox, {"visible": False}),
     "keyedit_precision_attention": OptionInfo(0.1, "Ctrl+up/down precision when editing (attention:1.1)", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001}),
     "keyedit_precision_extra": OptionInfo(0.05, "Ctrl+up/down precision when editing <extra networks:0.9>", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001}),
     "quicksettings": OptionInfo("sd_model_checkpoint", "Quicksettings list"),
@@ -417,7 +405,7 @@ options_templates.update(options_section(('ui', "Live previews"), {
 
 options_templates.update(options_section(('sampler-params', "Sampler parameters"), {
     "show_samplers": OptionInfo(["Euler a", "UniPC", "DDIM", "DPM++ SDE", "DPM++ SDE", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers()]}),
-    "fallback_sampler": OptionInfo("Euler a", "Fallback sampler if primary sampler is not compatible", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}),
+    "fallback_sampler": OptionInfo("Euler a", "Secondary sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}),
     "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),
     "eta_ddim": OptionInfo(0.0, "Noise multiplier for DDIM (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),
     "ddim_discretize": OptionInfo('uniform', "DDIM discretize img2img", gr.Radio, {"choices": ['uniform', 'quad']}),
diff --git a/modules/txt2img.py b/modules/txt2img.py
index 2fcb4c49d..17e5ce909 100644
--- a/modules/txt2img.py
+++ b/modules/txt2img.py
@@ -1,16 +1,15 @@
 import modules.scripts
 from modules import sd_samplers
 from modules.generation_parameters_copypaste import create_override_settings_dict
-from modules.processing import StableDiffusionProcessingTxt2Img, process_images
-from modules.shared import opts
-import modules.shared as shared
+from modules.processing import StableDiffusionProcessingTxt2Img, process_images, memory_stats
+from modules.shared import opts, sd_model, cmd_opts, log
 from modules.ui import plaintext_to_html
 
 
 def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, seed_enable_extras: bool, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, override_settings_texts, *args): # pylint: disable=unused-argument
     override_settings = create_override_settings_dict(override_settings_texts)
     p = StableDiffusionProcessingTxt2Img(
-        sd_model=shared.sd_model,
+        sd_model=sd_model,
         outpath_samples=opts.outdir_samples or opts.outdir_txt2img_samples,
         outpath_grids=opts.outdir_grids or opts.outdir_txt2img_grids,
         prompt=prompt,
@@ -46,8 +45,7 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step
     if processed is None:
         processed = process_images(p)
     p.close()
-    shared.total_tqdm.clear()
     generation_info_js = processed.js()
-    if opts.do_not_show_images:
-        processed.images = []
+    if cmd_opts.debug:
+        log.info(f'Processed: {len(processed.images)} Memory: {memory_stats()} txt')
     return processed.images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments)
diff --git a/scripts/postprocessing_upscale.py b/scripts/postprocessing_upscale.py
index b2f9c7408..9ee8878ad 100644
--- a/scripts/postprocessing_upscale.py
+++ b/scripts/postprocessing_upscale.py
@@ -1,9 +1,7 @@
 from PIL import Image
 import numpy as np
-
-from modules import scripts_postprocessing, shared
 import gradio as gr
-
+from modules import scripts_postprocessing, shared
 from modules.ui_components import FormRow, ToolButton
 from modules.ui import switch_values_symbol
 
@@ -15,7 +13,7 @@ class ScriptPostprocessingUpscale(scripts_postprocessing.ScriptPostprocessing):
     order = 1000
 
     def ui(self):
-        selected_tab = gr.State(value=0)
+        selected_tab = gr.State(value=0) # pylint: disable=abstract-class-instantiated
 
         with gr.Column():
             with FormRow():
@@ -80,7 +78,7 @@ class ScriptPostprocessingUpscale(scripts_postprocessing.ScriptPostprocessing):
 
         return image
 
-    def process(self, pp: scripts_postprocessing.PostprocessedImage, upscale_mode=1, upscale_by=2.0, upscale_to_width=None, upscale_to_height=None, upscale_crop=False, upscaler_1_name=None, upscaler_2_name=None, upscaler_2_visibility=0.0):
+    def process(self, pp: scripts_postprocessing.PostprocessedImage, upscale_mode=1, upscale_by=2.0, upscale_to_width=None, upscale_to_height=None, upscale_crop=False, upscaler_1_name=None, upscaler_2_name=None, upscaler_2_visibility=0.0): # pylint: disable=arguments-differ
         if upscaler_1_name == "None":
             upscaler_1_name = None
 
@@ -97,13 +95,13 @@ class ScriptPostprocessingUpscale(scripts_postprocessing.ScriptPostprocessing):
         assert upscaler2 or (upscaler_2_name is None), f'could not find upscaler named {upscaler_2_name}'
 
         upscaled_image = self.upscale(pp.image, pp.info, upscaler1, upscale_mode, upscale_by, upscale_to_width, upscale_to_height, upscale_crop)
-        pp.info[f"Postprocess upscaler"] = upscaler1.name
+        pp.info["Postprocess upscaler"] = upscaler1.name
 
         if upscaler2 and upscaler_2_visibility > 0:
             second_upscale = self.upscale(pp.image, pp.info, upscaler2, upscale_mode, upscale_by, upscale_to_width, upscale_to_height, upscale_crop)
             upscaled_image = Image.blend(upscaled_image, second_upscale, upscaler_2_visibility)
 
-            pp.info[f"Postprocess upscaler 2"] = upscaler2.name
+            pp.info["Postprocess upscaler 2"] = upscaler2.name
 
         pp.image = upscaled_image
 
@@ -125,7 +123,7 @@ class ScriptPostprocessingUpscaleSimple(ScriptPostprocessingUpscale):
             "upscaler_name": upscaler_name,
         }
 
-    def process(self, pp: scripts_postprocessing.PostprocessedImage, upscale_by=2.0, upscaler_name=None):
+    def process(self, pp: scripts_postprocessing.PostprocessedImage, upscale_by=2.0, upscaler_name=None): # pylint: disable=arguments-differ
         if upscaler_name is None or upscaler_name == "None":
             return
 
@@ -133,4 +131,4 @@ class ScriptPostprocessingUpscaleSimple(ScriptPostprocessingUpscale):
         assert upscaler1, f'could not find upscaler named {upscaler_name}'
 
         pp.image = self.upscale(pp.image, pp.info, upscaler1, 0, upscale_by, 0, 0, False)
-        pp.info[f"Postprocess upscaler"] = upscaler1.name
+        pp.info["Postprocess upscaler"] = upscaler1.name
diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py
index 748fe7bbc..700cc599e 100644
--- a/scripts/xyz_grid.py
+++ b/scripts/xyz_grid.py
@@ -587,7 +587,6 @@ class Script(scripts.Script):
         cell_console_text = f"; {image_cell_count} images per cell" if image_cell_count > 1 else ""
         plural_s = 's' if len(zs) > 1 else ''
         print(f"X/Y/Z plot will create {len(xs) * len(ys) * len(zs) * image_cell_count} images on {len(zs)} {len(xs)}x{len(ys)} grid{plural_s}{cell_console_text}. (Total steps to process: {total_steps})")
-        shared.total_tqdm.updateTotal(total_steps)
 
         state.xyz_plot_x = AxisInfo(x_opt, xs)
         state.xyz_plot_y = AxisInfo(y_opt, ys)
diff --git a/webui.py b/webui.py
index 54d159d24..7ca052a2e 100644
--- a/webui.py
+++ b/webui.py
@@ -105,7 +105,7 @@ def initialize():
     startup_timer.record("vae")
 
     shared.opts.onchange("sd_vae", wrap_queued_call(lambda: modules.sd_vae.reload_vae_weights()), call=False)
-    shared.opts.onchange("sd_vae_as_default", wrap_queued_call(lambda: modules.sd_vae.reload_vae_weights()), call=False)
+    # shared.opts.onchange("sd_vae_as_default", wrap_queued_call(lambda: modules.sd_vae.reload_vae_weights()), call=False)
     shared.opts.onchange("temp_dir", ui_tempdir.on_tmpdir_changed)
     shared.opts.onchange("gradio_theme", shared.reload_gradio_theme)
     startup_timer.record("opts onchange")

From 6cd4d62f710a5c49dc6ff46a5dfb9ca93c8c5246 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 17:36:13 -0400
Subject: [PATCH 34/69] update repos

---
 wiki | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wiki b/wiki
index 12603bcde..6cd8fde16 160000
--- a/wiki
+++ b/wiki
@@ -1 +1 @@
-Subproject commit 12603bcdec55df780b18612d58b6d0dcd4c27f96
+Subproject commit 6cd8fde165190057c0849fa6f8dbb183f717b176

From 5dc9743592f7d41eb5efd49c10792f63ab783d57 Mon Sep 17 00:00:00 2001
From: nekoworkshop <nekoworkshop@gmail.com>
Date: Sat, 29 Apr 2023 14:41:37 -0400
Subject: [PATCH 35/69] Initial implementation

---
 modules/generation_parameters_copypaste.py |  2 +-
 scripts/xyz_grid.py                        | 42 ++++++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py
index 964432d72..b6609e224 100644
--- a/modules/generation_parameters_copypaste.py
+++ b/modules/generation_parameters_copypaste.py
@@ -316,7 +316,7 @@ infotext_to_setting_name_mapping = [
     ('Token merging merge attention', 'token_merging_merge_attention'),
     ('Token merging merge cross attention', 'token_merging_merge_cross_attention'),
     ('Token merging merge mlp', 'token_merging_merge_mlp'),
-    ('Token merging maximum downsampling', 'token_merging_maximum_downsampling'),
+    ('Token merging maximum downsampling', 'token_merging_maximum_down_sampling'),
     ('Token merging stride x', 'token_merging_stride_x'),
     ('Token merging stride y', 'token_merging_stride_y')
 ]
diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py
index 700cc599e..68c294959 100644
--- a/scripts/xyz_grid.py
+++ b/scripts/xyz_grid.py
@@ -145,6 +145,39 @@ def apply_face_restore(p, opt, x):
 
     p.restore_faces = is_active
 
+def apply_token_merging1(p, x, xs):
+    is_active = x.lower() in ('true', 'yes', 'y', '1')
+    p.override_settings["token_merging"] = is_active
+
+def apply_token_merging_ratio_hr(p, x, xs):
+    p.override_settings["token_merging_ratio_hr"] = x
+
+def apply_token_merging_ratio(p, x, xs):
+    p.override_settings["token_merging_ratio"] = x
+
+def apply_token_merging_hr_only(p, x, xs):
+    is_active = x.lower() in ('true', 'yes', 'y', '1')
+    p.override_settings["token_merging_hr_only"] = is_active
+
+def apply_token_merging_random(p, x, xs):
+    is_active = x.lower() in ('true', 'yes', 'y', '1')
+    p.override_settings["token_merging_random"] = is_active
+
+def apply_token_merging_attention(p, x, xs):
+    is_active = x.lower() in ('true', 'yes', 'y', '1')
+    p.override_settings["token_merging_merge_attention"] = is_active
+
+def apply_token_merging_cross_attention(p, x, xs):
+    is_active = x.lower() in ('true', 'yes', 'y', '1')
+    p.override_settings["token_merging_merge_cross_attention"] = is_active
+
+def apply_token_merging_mlp(p, x, xs):
+    is_active = x.lower() in ('true', 'yes', 'y', '1')
+    p.override_settings["token_merging_merge_mlp"] = is_active
+
+def apply_token_merging_maximum_down_sampling (p, x, xs):
+    p.override_settings["token_merging_maximum_down_sampling"] = x
+    #opts.data["token_merging_maximum_down_sampling"] = x
 
 def format_value_add_label(p, opt, x):
     if type(x) == float:
@@ -226,6 +259,15 @@ axis_options = [
     AxisOption("Styles", str, apply_styles, choices=lambda: list(shared.prompt_styles.styles)),
     AxisOption("UniPC Order", int, apply_uni_pc_order, cost=0.5),
     AxisOption("Face restore", str, apply_face_restore, format_value=format_value),
+    AxisOption("Token Merging", str, apply_token_merging1),
+    AxisOption("Token merging ratio",float,apply_token_merging_ratio),
+    AxisOption("Token merging ratio for Hires fix",float,apply_token_merging_ratio_hr),
+    AxisOption("Token merging apply only to Hires fix",str,apply_token_merging_hr_only, choices= lambda: ["Yes","No"]),
+    AxisOption("Token Merging use random pertubations",str,apply_token_merging_random, choices = lambda: ["Yes","No"]),
+    AxisOption("Token Merging merge attention", str, apply_token_merging_attention, choices= lambda: ["Yes","No"]),
+    AxisOption("Token Merging merge cross attention", str, apply_token_merging_cross_attention, choices= lambda: ["Yes","No"]),
+    AxisOption("Token Merging merge mlp", str, apply_token_merging_mlp, choices= lambda: ["Yes","No"]),
+    AxisOption("Token Merging maxium down sampling", int, apply_token_merging_maximum_down_sampling, choices= lambda: ["1","2","4","8"])
 ]
 
 

From 0e165ed2ee45238133d0132c018280c97eb0f053 Mon Sep 17 00:00:00 2001
From: nekoworkshop <nekoworkshop@gmail.com>
Date: Sat, 29 Apr 2023 17:18:01 -0400
Subject: [PATCH 36/69] Use SharedSettingsStackHelper

---
 scripts/xyz_grid.py | 54 ++++++++++++++++++++++++++++++++-------------
 1 file changed, 39 insertions(+), 15 deletions(-)

diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py
index 68c294959..2d785bf04 100644
--- a/scripts/xyz_grid.py
+++ b/scripts/xyz_grid.py
@@ -145,39 +145,40 @@ def apply_face_restore(p, opt, x):
 
     p.restore_faces = is_active
 
-def apply_token_merging1(p, x, xs):
-    is_active = x.lower() in ('true', 'yes', 'y', '1')
-    p.override_settings["token_merging"] = is_active
-
 def apply_token_merging_ratio_hr(p, x, xs):
-    p.override_settings["token_merging_ratio_hr"] = x
+    opts.data["token_merging_ratio_hr"] = x
 
 def apply_token_merging_ratio(p, x, xs):
-    p.override_settings["token_merging_ratio"] = x
+    opts.data["token_merging_ratio"] = x
 
 def apply_token_merging_hr_only(p, x, xs):
     is_active = x.lower() in ('true', 'yes', 'y', '1')
-    p.override_settings["token_merging_hr_only"] = is_active
+    opts.data["token_merging_hr_only"] = is_active
 
 def apply_token_merging_random(p, x, xs):
     is_active = x.lower() in ('true', 'yes', 'y', '1')
-    p.override_settings["token_merging_random"] = is_active
+    opts.data["token_merging_random"] = is_active
 
 def apply_token_merging_attention(p, x, xs):
     is_active = x.lower() in ('true', 'yes', 'y', '1')
-    p.override_settings["token_merging_merge_attention"] = is_active
+    opts.data["token_merging_merge_attention"] = is_active
 
 def apply_token_merging_cross_attention(p, x, xs):
     is_active = x.lower() in ('true', 'yes', 'y', '1')
-    p.override_settings["token_merging_merge_cross_attention"] = is_active
+    opts.data["token_merging_merge_cross_attention"] = is_active
 
 def apply_token_merging_mlp(p, x, xs):
     is_active = x.lower() in ('true', 'yes', 'y', '1')
-    p.override_settings["token_merging_merge_mlp"] = is_active
+    opts.data["token_merging_merge_mlp"] = is_active
 
 def apply_token_merging_maximum_down_sampling (p, x, xs):
-    p.override_settings["token_merging_maximum_down_sampling"] = x
-    #opts.data["token_merging_maximum_down_sampling"] = x
+    opts.data["token_merging_maximum_down_sampling"] = x
+
+def apply_token_merging_stride_x(p, x, xs):
+    opts.data["token_merging_stride_x"] = x
+
+def apply_token_merging_stride_y(p, x, xs):
+    opts.data["token_merging_stride_y"] = x
 
 def format_value_add_label(p, opt, x):
     if type(x) == float:
@@ -259,7 +260,6 @@ axis_options = [
     AxisOption("Styles", str, apply_styles, choices=lambda: list(shared.prompt_styles.styles)),
     AxisOption("UniPC Order", int, apply_uni_pc_order, cost=0.5),
     AxisOption("Face restore", str, apply_face_restore, format_value=format_value),
-    AxisOption("Token Merging", str, apply_token_merging1),
     AxisOption("Token merging ratio",float,apply_token_merging_ratio),
     AxisOption("Token merging ratio for Hires fix",float,apply_token_merging_ratio_hr),
     AxisOption("Token merging apply only to Hires fix",str,apply_token_merging_hr_only, choices= lambda: ["Yes","No"]),
@@ -267,7 +267,9 @@ axis_options = [
     AxisOption("Token Merging merge attention", str, apply_token_merging_attention, choices= lambda: ["Yes","No"]),
     AxisOption("Token Merging merge cross attention", str, apply_token_merging_cross_attention, choices= lambda: ["Yes","No"]),
     AxisOption("Token Merging merge mlp", str, apply_token_merging_mlp, choices= lambda: ["Yes","No"]),
-    AxisOption("Token Merging maxium down sampling", int, apply_token_merging_maximum_down_sampling, choices= lambda: ["1","2","4","8"])
+    AxisOption("Token Merging maxium down sampling", int, apply_token_merging_maximum_down_sampling, choices= lambda: ["1","2","4","8"]),
+    AxisOption("Token Merging Stride - X", int, apply_token_merging_stride_x, choices= lambda: ["2","4","6","8"]),
+    AxisOption("Token Merging Stride - Y", int, apply_token_merging_stride_y, choices= lambda: ["2","4","6","8"])
 ]
 
 
@@ -384,11 +386,23 @@ def draw_xyz_grid(p, xs, ys, zs, x_labels, y_labels, z_labels, cell, draw_legend
 
 class SharedSettingsStackHelper(object):
     def __enter__(self):
+        #Save overridden settings so they can be restored later.
         self.CLIP_stop_at_last_layers = opts.CLIP_stop_at_last_layers
         self.vae = opts.sd_vae
         self.uni_pc_order = opts.uni_pc_order
+        self.token_merging_ratio_hr = opts.token_merging_ratio_hr
+        self.token_merging_ratio = opts.token_merging_ratio
+        self.token_merging_hr_only = opts.token_merging_hr_only
+        self.token_merging_random = opts.token_merging_random
+        self.token_merging_merge_attention = opts.token_merging_merge_attention
+        self.token_merging_merge_cross_attention = opts.token_merging_merge_cross_attention
+        self.token_merging_merge_mlp = opts.token_merging_merge_mlp
+        self.token_merging_maximum_down_sampling = opts.token_merging_maximum_down_sampling
+        self.token_merging_stride_x = opts.token_merging_stride_x
+        self.token_merging_stride_y = opts.token_merging_stride_y
 
     def __exit__(self, exc_type, exc_value, tb):
+        #Restore overriden settings after plot generation.
         opts.data["sd_vae"] = self.vae
         opts.data["uni_pc_order"] = self.uni_pc_order
         sd_models.reload_model_weights()
@@ -396,6 +410,16 @@ class SharedSettingsStackHelper(object):
 
         opts.data["CLIP_stop_at_last_layers"] = self.CLIP_stop_at_last_layers
 
+        opts.data["token_merging_ratio_hr"] = self.token_merging_ratio_hr
+        opts.data["token_merging_ratio"] = self.token_merging_ratio
+        opts.data["token_merging_hr_only"] = self.token_merging_hr_only
+        opts.data["token_merging_random"] = self.token_merging_random
+        opts.data["token_merging_merge_attention"] = self.token_merging_merge_attention
+        opts.data["token_merging_merge_cross_attention"] = self.token_merging_merge_cross_attention
+        opts.data["token_merging_merge_mlp"] = self.token_merging_merge_mlp
+        opts.data["token_merging_maximum_down_sampling"] = self.token_merging_maximum_down_sampling
+        opts.data["token_merging_stride_x"] = self.token_merging_stride_x
+        opts.data["token_merging_stride_y"] = self.token_merging_stride_y
 
 re_range = re.compile(r"\s*([+-]?\s*\d+)\s*-\s*([+-]?\s*\d+)(?:\s*\(([+-]\d+)\s*\))?\s*")
 re_range_float = re.compile(r"\s*([+-]?\s*\d+(?:.\d*)?)\s*-\s*([+-]?\s*\d+(?:.\d*)?)(?:\s*\(([+-]\d+(?:.\d*)?)\s*\))?\s*")

From e97443ff1b9bc76d48f5eee1e8d8c88a8a6b23f3 Mon Sep 17 00:00:00 2001
From: nekoworkshop <nekoworkshop@gmail.com>
Date: Sat, 29 Apr 2023 17:37:58 -0400
Subject: [PATCH 37/69] Adjust axis names to be shorter.

---
 scripts/xyz_grid.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py
index 2d785bf04..f47f0d167 100644
--- a/scripts/xyz_grid.py
+++ b/scripts/xyz_grid.py
@@ -260,16 +260,16 @@ axis_options = [
     AxisOption("Styles", str, apply_styles, choices=lambda: list(shared.prompt_styles.styles)),
     AxisOption("UniPC Order", int, apply_uni_pc_order, cost=0.5),
     AxisOption("Face restore", str, apply_face_restore, format_value=format_value),
-    AxisOption("Token merging ratio",float,apply_token_merging_ratio),
-    AxisOption("Token merging ratio for Hires fix",float,apply_token_merging_ratio_hr),
-    AxisOption("Token merging apply only to Hires fix",str,apply_token_merging_hr_only, choices= lambda: ["Yes","No"]),
-    AxisOption("Token Merging use random pertubations",str,apply_token_merging_random, choices = lambda: ["Yes","No"]),
-    AxisOption("Token Merging merge attention", str, apply_token_merging_attention, choices= lambda: ["Yes","No"]),
-    AxisOption("Token Merging merge cross attention", str, apply_token_merging_cross_attention, choices= lambda: ["Yes","No"]),
-    AxisOption("Token Merging merge mlp", str, apply_token_merging_mlp, choices= lambda: ["Yes","No"]),
-    AxisOption("Token Merging maxium down sampling", int, apply_token_merging_maximum_down_sampling, choices= lambda: ["1","2","4","8"]),
-    AxisOption("Token Merging Stride - X", int, apply_token_merging_stride_x, choices= lambda: ["2","4","6","8"]),
-    AxisOption("Token Merging Stride - Y", int, apply_token_merging_stride_y, choices= lambda: ["2","4","6","8"])
+    AxisOption("ToMe ratio",float,apply_token_merging_ratio),
+    AxisOption("ToMe ratio for Hires fix",float,apply_token_merging_ratio_hr),
+    AxisOption("ToMe apply only to Hires fix",str,apply_token_merging_hr_only, choices= lambda: ["Yes","No"]),
+    AxisOption("ToMe random pertubations",str,apply_token_merging_random, choices = lambda: ["Yes","No"]),
+    AxisOption("ToMe merge attention", str, apply_token_merging_attention, choices= lambda: ["Yes","No"]),
+    AxisOption("ToMe merge cross attention", str, apply_token_merging_cross_attention, choices= lambda: ["Yes","No"]),
+    AxisOption("ToMe merge mlp", str, apply_token_merging_mlp, choices= lambda: ["Yes","No"]),
+    AxisOption("ToMe maximum down sampling", int, apply_token_merging_maximum_down_sampling, choices= lambda: ["1","2","4","8"]),
+    AxisOption("ToMe Stride - X", int, apply_token_merging_stride_x, choices= lambda: ["2","4","6","8"]),
+    AxisOption("ToMe Stride - Y", int, apply_token_merging_stride_y, choices= lambda: ["2","4","6","8"])
 ]
 
 

From c4936fc92784c452bb067b5097b54476400b1abc Mon Sep 17 00:00:00 2001
From: nekoworkshop <nekoworkshop@gmail.com>
Date: Sat, 29 Apr 2023 17:59:00 -0400
Subject: [PATCH 38/69] Extra information in ToMe related settings

---
 modules/shared.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/modules/shared.py b/modules/shared.py
index 0dfefa8c4..dc8ac5067 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -422,14 +422,14 @@ options_templates.update(options_section(('sampler-params', "Sampler parameters"
 
 options_templates.update(options_section(('token_merging', 'Token Merging'), {
     "token_merging": OptionInfo(False, "Enable redundant token merging via tomesd. This can provide significant speed and memory improvements.", gr.Checkbox),
-    "token_merging_ratio": OptionInfo(0.5, "Merging Ratio", gr.Slider, {"minimum": 0, "maximum": 0.9, "step": 0.1}),
+    "token_merging_ratio": OptionInfo(0.5, "Merging Ratio. Higher merging ratio = faster generation, smaller VRAM usage, lower quality.", gr.Slider, {"minimum": 0, "maximum": 0.9, "step": 0.1}),
     "token_merging_hr_only": OptionInfo(True, "Apply only to high-res fix pass. Disabling can yield a ~20-35% speedup on contemporary resolutions.", gr.Checkbox),
     "token_merging_ratio_hr": OptionInfo(0.5, "Merging Ratio (high-res pass) - If 'Apply only to high-res' is enabled, this will always be the ratio used.", gr.Slider, {"minimum": 0, "maximum": 0.9, "step": 0.1}),
     "token_merging_random": OptionInfo(False, "Use random perturbations - Can improve outputs for certain samplers. For others, it may cause visual artifacting.", gr.Checkbox),
-    "token_merging_merge_attention": OptionInfo(True, "Merge attention", gr.Checkbox),
-    "token_merging_merge_cross_attention": OptionInfo(False, "Merge cross attention", gr.Checkbox),
-    "token_merging_merge_mlp": OptionInfo(False, "Merge mlp", gr.Checkbox),
-    "token_merging_maximum_down_sampling": OptionInfo(1, "Maximum down sampling", gr.Dropdown, lambda: {"choices": ["1", "2", "4", "8"]}),
+    "token_merging_merge_attention": OptionInfo(True, "Merge attention (Recommend on)", gr.Checkbox),
+    "token_merging_merge_cross_attention": OptionInfo(False, "Merge cross attention (Recommend off)", gr.Checkbox),
+    "token_merging_merge_mlp": OptionInfo(False, "Merge mlp (Strongly recommend off)", gr.Checkbox),
+    "token_merging_maximum_down_sampling": OptionInfo(1, "Maximum down sampling", gr.Radio, lambda: {"choices": [1, 2, 4, 8]}),
     "token_merging_stride_x": OptionInfo(2, "Stride - X", gr.Slider, {"minimum": 2, "maximum": 8, "step": 2}),
     "token_merging_stride_y": OptionInfo(2, "Stride - Y", gr.Slider, {"minimum": 2, "maximum": 8, "step": 2})
 }))

From 1469f8c69c8964be07fec0c67b554d9d22a11371 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 18:18:08 -0400
Subject: [PATCH 39/69] force lightbox

---
 javascript/imageviewer.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/javascript/imageviewer.js b/javascript/imageviewer.js
index c928d9c5f..149177430 100644
--- a/javascript/imageviewer.js
+++ b/javascript/imageviewer.js
@@ -105,8 +105,8 @@ function setupImageForLightbox(e) {
 	var isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1
 	var event = isFirefox ? 'mousedown' : 'click'
 	e.addEventListener(event, function (evt) {
-		if(!opts.js_modal_lightbox || evt.button != 0) return;
-		modalZoomSet(gradioApp().getElementById('modalImage'), opts.js_modal_lightbox_initially_zoomed)
+		if (evt.button != 0) return;
+		modalZoomSet(gradioApp().getElementById('modalImage'), true)
 		evt.preventDefault()
 		showModal(evt)
 	}, true);

From 1f5d5a17e39a901c1f81a814026e17100dcf0d0c Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 18:27:21 -0400
Subject: [PATCH 40/69] update lint rules

---
 .github/workflows/on_pull_request.yaml | 4 ----
 .pylintrc                              | 3 +++
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml
index 011fe9b2d..8693fd003 100644
--- a/.github/workflows/on_pull_request.yaml
+++ b/.github/workflows/on_pull_request.yaml
@@ -28,11 +28,7 @@ jobs:
         run: | 
           python -m pip install --upgrade pip
           pip install pylint
-      # This lets PyLint check to see if it can resolve imports
       - name: Install dependencies
-        run: |
-          export COMMANDLINE_ARGS="--skip-torch-cuda-test --exit"
-          python launch.py
       - name: Analysing the code with pylint
         run: |
           pylint $(git ls-files '*.py')
diff --git a/.pylintrc b/.pylintrc
index 72e1224e8..515dce01d 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -11,6 +11,7 @@ fail-under=10
 ignore=CVS
 ignore-paths=^repositories/.*$,
              ^extensions/.*$,
+             ^extensions-builtin/.*$,
              /usr/lib/.*$,
 ignore-patterns=
 ignored-modules=
@@ -141,6 +142,8 @@ disable=raw-checker-failed,
         consider-using-dict-items,
         dangerous-default-value,
         unnecessary-dunder-call,
+        invalid-name,
+        R0801,
 enable=c-extension-no-member
 
 [METHOD_ARGS]

From f813e6022b1e8471044fc38edbf5602a5635520d Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 18:38:13 -0400
Subject: [PATCH 41/69] add test mode

---
 .github/workflows/on_pull_request.yaml | 10 +++++-----
 launch.py                              |  8 ++++++--
 setup.py                               |  1 +
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml
index 8693fd003..57f46ab6e 100644
--- a/.github/workflows/on_pull_request.yaml
+++ b/.github/workflows/on_pull_request.yaml
@@ -24,11 +24,11 @@ jobs:
           python-version: 3.10.6
           cache: pip
           cache-dependency-path: requirements.txt
-      - name: Install PyLint
-        run: | 
-          python -m pip install --upgrade pip
-          pip install pylint
       - name: Install dependencies
+        run: |
+          python launch.py --test
       - name: Analysing the code with pylint
         run: |
-          pylint $(git ls-files '*.py')
+        python -m pip install --upgrade pip
+        pip install pylint
+        pylint $(git ls-files '*.py')
diff --git a/launch.py b/launch.py
index bf34ee507..ad55ca2e3 100644
--- a/launch.py
+++ b/launch.py
@@ -97,5 +97,9 @@ if __name__ == "__main__":
     setup.log.info(f"Server arguments: {sys.argv[1:]}")
     setup.log.debug('Starting WebUI')
     logging.disable(logging.INFO)
-    import webui
-    webui.webui()
+    if args.test:
+        setup.log.info(f"Test only")
+        import webui
+        exit(0)
+    else:
+        webui.webui()
diff --git a/setup.py b/setup.py
index 804ee35b0..649718bd0 100644
--- a/setup.py
+++ b/setup.py
@@ -496,6 +496,7 @@ def add_args():
     parser.add_argument('--skip-extensions', default = False, action='store_true', help = "Skips running individual extension installers, default: %(default)s")
     parser.add_argument('--skip-git', default = False, action='store_true', help = "Skips running all GIT operations, default: %(default)s")
     parser.add_argument('--experimental', default = False, action='store_true', help = "Allow unsupported versions of libraries, default: %(default)s")
+    parser.add_argument('--test', default = False, action='store_true', help = "Run test only, default: %(default)s")
 
 
 def parse_args():

From ddd6125e56940eb4b2d4580dce65ec423d8c7f77 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 18:44:03 -0400
Subject: [PATCH 42/69] set git triggers

---
 .github/workflows/on_pull_request.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml
index 57f46ab6e..b9e0eb7db 100644
--- a/.github/workflows/on_pull_request.yaml
+++ b/.github/workflows/on_pull_request.yaml
@@ -29,6 +29,6 @@ jobs:
           python launch.py --test
       - name: Analysing the code with pylint
         run: |
-        python -m pip install --upgrade pip
-        pip install pylint
-        pylint $(git ls-files '*.py')
+          python -m pip install --upgrade pip
+          pip install pylint
+          pylint $(git ls-files '*.py')

From 21b6e11b01433a4cc86fbe008d8dc06892b9d169 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 18:53:00 -0400
Subject: [PATCH 43/69] critical

---
 launch.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/launch.py b/launch.py
index ad55ca2e3..f80e9ae72 100644
--- a/launch.py
+++ b/launch.py
@@ -102,4 +102,5 @@ if __name__ == "__main__":
         import webui
         exit(0)
     else:
+        import webui
         webui.webui()

From d4fd25a7fdc350e7edde601443279786396c1abf Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 18:58:10 -0400
Subject: [PATCH 44/69] add debug mode

---
 launch.py          | 2 +-
 modules/img2img.py | 4 ++--
 modules/txt2img.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/launch.py b/launch.py
index f80e9ae72..995f61894 100644
--- a/launch.py
+++ b/launch.py
@@ -98,7 +98,7 @@ if __name__ == "__main__":
     setup.log.debug('Starting WebUI')
     logging.disable(logging.INFO)
     if args.test:
-        setup.log.info(f"Test only")
+        setup.log.info("Test only")
         import webui
         exit(0)
     else:
diff --git a/modules/img2img.py b/modules/img2img.py
index de889d4b7..fc7165751 100644
--- a/modules/img2img.py
+++ b/modules/img2img.py
@@ -60,7 +60,7 @@ def process_batch(p, input_dir, output_dir, inpaint_mask_dir, args):
                 if processed_image.mode == 'RGBA':
                     processed_image = processed_image.convert("RGB")
                 processed_image.save(os.path.join(output_dir, filename))
-    if cmd_opts.debug:
+    if cmd_opts.get('debug', False):
         log.info(f'Processed: {len(images)} Memory: {memory_stats()} batch')
 
 
@@ -146,6 +146,6 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s
             processed = process_images(p)
     p.close()
     generation_info_js = processed.js()
-    if cmd_opts.debug:
+    if cmd_opts.get('debug', False):
         log.info(f'Processed: {len(processed.images)} Memory: {memory_stats()} img')
     return processed.images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments)
diff --git a/modules/txt2img.py b/modules/txt2img.py
index 17e5ce909..45541749b 100644
--- a/modules/txt2img.py
+++ b/modules/txt2img.py
@@ -46,6 +46,6 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step
         processed = process_images(p)
     p.close()
     generation_info_js = processed.js()
-    if cmd_opts.debug:
+    if cmd_opts.get('debug', False):
         log.info(f'Processed: {len(processed.images)} Memory: {memory_stats()} txt')
     return processed.images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments)

From 9ac7f3771a2b8b488609709ffbf559d8c88ffd01 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 29 Apr 2023 19:30:26 -0400
Subject: [PATCH 45/69] update cmdflags

---
 .github/workflows/on_pull_request.yaml | 7 ++++---
 TODO.md                                | 5 ++---
 launch.py                              | 5 ++---
 modules/cmd_args.py                    | 1 +
 modules/img2img.py                     | 4 ++--
 modules/txt2img.py                     | 2 +-
 setup.py                               | 3 +--
 7 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml
index b9e0eb7db..6cb1005b0 100644
--- a/.github/workflows/on_pull_request.yaml
+++ b/.github/workflows/on_pull_request.yaml
@@ -24,10 +24,11 @@ jobs:
           python-version: 3.10.6
           cache: pip
           cache-dependency-path: requirements.txt
-      - name: Install dependencies
+      - name: Test Startup
         run: |
-          python launch.py --test
-      - name: Analysing the code with pylint
+          export COMMANDLINE_ARGS="--debug --test"
+          python launch.py
+      - name: Linting
         run: |
           python -m pip install --upgrade pip
           pip install pylint
diff --git a/TODO.md b/TODO.md
index 167a823ec..0a5b0e99d 100644
--- a/TODO.md
+++ b/TODO.md
@@ -4,10 +4,7 @@
 
 Stuff to be fixed...
 
-- Run VAE with hires at 1280
-- Transformers version
 - Move Restart Server from WebUI to Launch and reload modules
-- Follow-up on `p.script_args`
 - Mdularize `cli` scripts
 
 ## Features
@@ -16,9 +13,11 @@ Stuff to be added...
 
 - Update README
 - Add Gradio theme maker
+- Transformers version
 - Create new GitHub hooks/actions for CI/CD  
 - Redo Extensions tab: see <https://vladmandic.github.io/sd-extension-manager/pages/extensions.html>
 - Stream-load models as option for slow storage
+- Auto-test `torch.layer_norm` for FP16
 
 ## Investigate
 
diff --git a/launch.py b/launch.py
index 995f61894..8df3f787c 100644
--- a/launch.py
+++ b/launch.py
@@ -101,6 +101,5 @@ if __name__ == "__main__":
         setup.log.info("Test only")
         import webui
         exit(0)
-    else:
-        import webui
-        webui.webui()
+    import webui
+    webui.webui()
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index 3bca9b357..9b8438800 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -74,6 +74,7 @@ def compatibility_args(opts, args):
     parser.add_argument("--sub-quad-q-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_q_chunk_size)
     parser.add_argument("--sub-quad-kv-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_kv_chunk_size)
     parser.add_argument("--sub-quad-chunk-threshold", help=argparse.SUPPRESS, default=opts.sub_quad_chunk_threshold)
+    parser.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
 
     opts.use_old_emphasis_implementation = False
     opts.use_old_karras_scheduler_sigmas = False
diff --git a/modules/img2img.py b/modules/img2img.py
index fc7165751..de889d4b7 100644
--- a/modules/img2img.py
+++ b/modules/img2img.py
@@ -60,7 +60,7 @@ def process_batch(p, input_dir, output_dir, inpaint_mask_dir, args):
                 if processed_image.mode == 'RGBA':
                     processed_image = processed_image.convert("RGB")
                 processed_image.save(os.path.join(output_dir, filename))
-    if cmd_opts.get('debug', False):
+    if cmd_opts.debug:
         log.info(f'Processed: {len(images)} Memory: {memory_stats()} batch')
 
 
@@ -146,6 +146,6 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s
             processed = process_images(p)
     p.close()
     generation_info_js = processed.js()
-    if cmd_opts.get('debug', False):
+    if cmd_opts.debug:
         log.info(f'Processed: {len(processed.images)} Memory: {memory_stats()} img')
     return processed.images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments)
diff --git a/modules/txt2img.py b/modules/txt2img.py
index 45541749b..17e5ce909 100644
--- a/modules/txt2img.py
+++ b/modules/txt2img.py
@@ -46,6 +46,6 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step
         processed = process_images(p)
     p.close()
     generation_info_js = processed.js()
-    if cmd_opts.get('debug', False):
+    if cmd_opts.debug:
         log.info(f'Processed: {len(processed.images)} Memory: {memory_stats()} txt')
     return processed.images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments)
diff --git a/setup.py b/setup.py
index 649718bd0..906eaf136 100644
--- a/setup.py
+++ b/setup.py
@@ -487,8 +487,7 @@ def check_timestamp():
 
 
 def add_args():
-    if vars(parser)['_option_string_actions'].get('--debug', None) is None:
-        parser.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
+    parser.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
     parser.add_argument('--reset', default = False, action='store_true', help = "Reset main repository to latest version, default: %(default)s")
     parser.add_argument('--upgrade', default = False, action='store_true', help = "Upgrade main repository to latest version, default: %(default)s")
     parser.add_argument('--noupdate', default = False, action='store_true', help = "Skip update of extensions and submodules, default: %(default)s")

From e65d74100fe1990affd1048dc26f5eed92729a0e Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Sun, 30 Apr 2023 11:12:47 +0900
Subject: [PATCH 46/69] Move/Remove hijacks. Add warning.

---
 modules/dml/hijack/__init__.py  | 1 +
 modules/sd_hijack.py            | 3 ---
 modules/sd_hijack_inpainting.py | 1 -
 modules/shared.py               | 2 +-
 4 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/modules/dml/hijack/__init__.py b/modules/dml/hijack/__init__.py
index dd71784c0..d8cc0913a 100644
--- a/modules/dml/hijack/__init__.py
+++ b/modules/dml/hijack/__init__.py
@@ -2,3 +2,4 @@ import modules.dml.hijack.kdiffusion
 import modules.dml.hijack.stablediffusion
 import modules.dml.hijack.torch
 import modules.dml.hijack.realesrgan_model
+import modules.dml.hijack.plms
diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index 4a68216df..f817b7afd 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -257,9 +257,6 @@ class EmbeddingsWithFixes(torch.nn.Module):
             for offset, embedding in fixes:
                 emb = devices.cond_cast_unet(embedding.vec)
                 emb_len = min(tensor.shape[0] - offset - 1, emb.shape[0])
-                # DML Solution: type mismatch on half mode
-                if tensor.dtype == torch.float16 and emb.dtype == torch.float32 and not shared.cmd_opts.no_half:
-                    emb = emb.half()
                 tensor = torch.cat([tensor[0:offset + 1], emb[0:emb_len], tensor[offset + 1 + emb_len:]])
 
             vecs.append(tensor)
diff --git a/modules/sd_hijack_inpainting.py b/modules/sd_hijack_inpainting.py
index 3405e8a40..4b23c132d 100644
--- a/modules/sd_hijack_inpainting.py
+++ b/modules/sd_hijack_inpainting.py
@@ -53,7 +53,6 @@ def p_sample_plms(self, x, c, t, index, repeat_noise=False, use_original_steps=F
 
     def get_x_prev_and_pred_x0(e_t, index):
         # select parameters corresponding to the currently considered timestep
-        print(alphas[index]) # DML Solution: PLMS Sampling does not work without this print.
         a_t = torch.full((b, 1, 1, 1), alphas[index], device=device)
         a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device)
         sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device)
diff --git a/modules/shared.py b/modules/shared.py
index d8a3f9d64..7c5916cfd 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -322,7 +322,7 @@ options_templates.update(options_section(('cuda', "CUDA Settings"), {
     "memmon_poll_rate": OptionInfo(2, "VRAM usage polls per second during generation. Set to 0 to disable.", gr.Slider, {"minimum": 0, "maximum": 40, "step": 1}),
     "precision": OptionInfo("Autocast", "Precision type", gr.Radio, lambda: {"choices": ["Autocast", "Full"]}),
     "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" else "FP16", "Device precision type", gr.Radio, lambda: {"choices": ["FP32", "FP16", "BF16"]}),
-    "no_half": OptionInfo(True if is_device_dml else False, "Use full precision for model (--no-half)"),
+    "no_half": OptionInfo(True if is_device_dml else False, "Use full precision for model (--no-half)", None, None, lambda: print("Warning: Most of DirectML devices do not fully support half mode. Recommend to use full precision to model.") if is_device_dml else None),
     "no_half_vae": OptionInfo(True if is_device_dml else False, "Use full precision for VAE (--no-half-vae)"),
     "upcast_sampling": OptionInfo(True if sys.platform == "darwin" else False, "Enable upcast sampling. Usually produces similar results to --no-half with better performance while using less memory"),
     "disable_nan_check": OptionInfo(True, "Do not check if produced images/latent spaces have NaN values"),

From 93d638d8b8b7d5226a8fa56e6d9b96c7710da994 Mon Sep 17 00:00:00 2001
From: Seunghoon Lee <lshqqytiger@naver.com>
Date: Sun, 30 Apr 2023 11:15:09 +0900
Subject: [PATCH 47/69] Approx NN works fine.

---
 modules/dml/hijack/plms.py | 91 ++++++++++++++++++++++++++++++++++++++
 modules/shared.py          |  2 +-
 2 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 modules/dml/hijack/plms.py

diff --git a/modules/dml/hijack/plms.py b/modules/dml/hijack/plms.py
new file mode 100644
index 000000000..49979db28
--- /dev/null
+++ b/modules/dml/hijack/plms.py
@@ -0,0 +1,91 @@
+import torch
+
+from ldm.models.diffusion.ddim import noise_like
+
+import modules.sd_hijack_inpainting as plms_hijack
+
+
+@torch.no_grad()
+def p_sample_plms(self, x, c, t, index, repeat_noise=False, use_original_steps=False, quantize_denoised=False,
+                  temperature=1., noise_dropout=0., score_corrector=None, corrector_kwargs=None,
+                  unconditional_guidance_scale=1., unconditional_conditioning=None, old_eps=None, t_next=None, dynamic_threshold=None):
+    b, *_, device = *x.shape, x.device
+
+    def get_model_output(x, t):
+        if unconditional_conditioning is None or unconditional_guidance_scale == 1.:
+            e_t = self.model.apply_model(x, t, c)
+        else:
+            x_in = torch.cat([x] * 2)
+            t_in = torch.cat([t] * 2)
+
+            if isinstance(c, dict):
+                assert isinstance(unconditional_conditioning, dict)
+                c_in = dict()
+                for k in c:
+                    if isinstance(c[k], list):
+                        c_in[k] = [
+                            torch.cat([unconditional_conditioning[k][i], c[k][i]])
+                            for i in range(len(c[k]))
+                        ]
+                    else:
+                        c_in[k] = torch.cat([unconditional_conditioning[k], c[k]])
+            else:
+                c_in = torch.cat([unconditional_conditioning, c])
+
+            e_t_uncond, e_t = self.model.apply_model(x_in, t_in, c_in).chunk(2)
+            e_t = e_t_uncond + unconditional_guidance_scale * (e_t - e_t_uncond)
+
+        if score_corrector is not None:
+            assert self.model.parameterization == "eps"
+            e_t = score_corrector.modify_score(self.model, e_t, x, t, c, **corrector_kwargs)
+
+        return e_t
+
+    alphas = self.model.alphas_cumprod if use_original_steps else self.ddim_alphas
+    alphas_prev = self.model.alphas_cumprod_prev if use_original_steps else self.ddim_alphas_prev
+    sqrt_one_minus_alphas = self.model.sqrt_one_minus_alphas_cumprod if use_original_steps else self.ddim_sqrt_one_minus_alphas
+    sigmas = self.model.ddim_sigmas_for_original_num_steps if use_original_steps else self.ddim_sigmas
+
+    def get_x_prev_and_pred_x0(e_t, index):
+        # select parameters corresponding to the currently considered timestep
+        print(alphas[index]) # DML Solution: PLMS Sampling does not work without this print.
+        a_t = torch.full((b, 1, 1, 1), alphas[index], device=device)
+        a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device)
+        sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device)
+        sqrt_one_minus_at = torch.full((b, 1, 1, 1), sqrt_one_minus_alphas[index],device=device)
+
+        # current prediction for x_0
+        pred_x0 = (x - sqrt_one_minus_at * e_t) / a_t.sqrt()
+        if quantize_denoised:
+            pred_x0, _, *_ = self.model.first_stage_model.quantize(pred_x0)
+        if dynamic_threshold is not None:
+            from ldm.models.diffusion.sampling_util import norm_thresholding
+            pred_x0 = norm_thresholding(pred_x0, dynamic_threshold)
+        # direction pointing to x_t
+        dir_xt = (1. - a_prev - sigma_t**2).sqrt() * e_t
+        noise = sigma_t * noise_like(x.shape, device, repeat_noise) * temperature
+        if noise_dropout > 0.:
+            noise = torch.nn.functional.dropout(noise, p=noise_dropout)
+        x_prev = a_prev.sqrt() * pred_x0 + dir_xt + noise
+        return x_prev, pred_x0
+
+    e_t = get_model_output(x, t)
+    if len(old_eps) == 0:
+        # Pseudo Improved Euler (2nd order)
+        x_prev, pred_x0 = get_x_prev_and_pred_x0(e_t, index)
+        e_t_next = get_model_output(x_prev, t_next)
+        e_t_prime = (e_t + e_t_next) / 2
+    elif len(old_eps) == 1:
+        # 2nd order Pseudo Linear Multistep (Adams-Bashforth)
+        e_t_prime = (3 * e_t - old_eps[-1]) / 2
+    elif len(old_eps) == 2:
+        # 3nd order Pseudo Linear Multistep (Adams-Bashforth)
+        e_t_prime = (23 * e_t - 16 * old_eps[-1] + 5 * old_eps[-2]) / 12
+    elif len(old_eps) >= 3:
+        # 4nd order Pseudo Linear Multistep (Adams-Bashforth)
+        e_t_prime = (55 * e_t - 59 * old_eps[-1] + 37 * old_eps[-2] - 9 * old_eps[-3]) / 24
+
+    x_prev, pred_x0 = get_x_prev_and_pred_x0(e_t_prime, index)
+
+    return x_prev, pred_x0, e_t
+plms_hijack.p_sample_plms = p_sample_plms
diff --git a/modules/shared.py b/modules/shared.py
index 7c5916cfd..ee8ca7f0a 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -416,7 +416,7 @@ options_templates.update(options_section(('ui', "Live previews"), {
     "live_previews_enable": OptionInfo(True, "Show live previews of the created image"),
     "show_progress_grid": OptionInfo(True, "Show previews of all images generated in a batch as a grid"),
     "show_progress_every_n_steps": OptionInfo(1, "Show new live preview image every N sampling steps. Set to -1 to show after completion of batch.", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}),
-    "show_progress_type": OptionInfo("Approx cheap" if is_device_dml else "Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}), # DML Solution: Use Approx cheap instead of Approx NN as a default progress type.
+    "show_progress_type": OptionInfo("Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}),
     "live_preview_content": OptionInfo("Combined", "Live preview subject", gr.Radio, {"choices": ["Combined", "Prompt", "Negative prompt"]}),
     "live_preview_refresh_period": OptionInfo(250, "Progressbar/preview update period, in milliseconds")
 }))

From df965a837bb6374b25b4f274b752805d68e38634 Mon Sep 17 00:00:00 2001
From: nekoworkshop <nekoworkshop@gmail.com>
Date: Sat, 29 Apr 2023 23:01:31 -0400
Subject: [PATCH 48/69] Remove less useful ToMe options from the xyz plot

---
 scripts/xyz_grid.py | 48 +--------------------------------------------
 1 file changed, 1 insertion(+), 47 deletions(-)

diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py
index f47f0d167..d216d2fef 100644
--- a/scripts/xyz_grid.py
+++ b/scripts/xyz_grid.py
@@ -151,35 +151,10 @@ def apply_token_merging_ratio_hr(p, x, xs):
 def apply_token_merging_ratio(p, x, xs):
     opts.data["token_merging_ratio"] = x
 
-def apply_token_merging_hr_only(p, x, xs):
-    is_active = x.lower() in ('true', 'yes', 'y', '1')
-    opts.data["token_merging_hr_only"] = is_active
-
 def apply_token_merging_random(p, x, xs):
     is_active = x.lower() in ('true', 'yes', 'y', '1')
     opts.data["token_merging_random"] = is_active
 
-def apply_token_merging_attention(p, x, xs):
-    is_active = x.lower() in ('true', 'yes', 'y', '1')
-    opts.data["token_merging_merge_attention"] = is_active
-
-def apply_token_merging_cross_attention(p, x, xs):
-    is_active = x.lower() in ('true', 'yes', 'y', '1')
-    opts.data["token_merging_merge_cross_attention"] = is_active
-
-def apply_token_merging_mlp(p, x, xs):
-    is_active = x.lower() in ('true', 'yes', 'y', '1')
-    opts.data["token_merging_merge_mlp"] = is_active
-
-def apply_token_merging_maximum_down_sampling (p, x, xs):
-    opts.data["token_merging_maximum_down_sampling"] = x
-
-def apply_token_merging_stride_x(p, x, xs):
-    opts.data["token_merging_stride_x"] = x
-
-def apply_token_merging_stride_y(p, x, xs):
-    opts.data["token_merging_stride_y"] = x
-
 def format_value_add_label(p, opt, x):
     if type(x) == float:
         x = round(x, 8)
@@ -262,14 +237,7 @@ axis_options = [
     AxisOption("Face restore", str, apply_face_restore, format_value=format_value),
     AxisOption("ToMe ratio",float,apply_token_merging_ratio),
     AxisOption("ToMe ratio for Hires fix",float,apply_token_merging_ratio_hr),
-    AxisOption("ToMe apply only to Hires fix",str,apply_token_merging_hr_only, choices= lambda: ["Yes","No"]),
-    AxisOption("ToMe random pertubations",str,apply_token_merging_random, choices = lambda: ["Yes","No"]),
-    AxisOption("ToMe merge attention", str, apply_token_merging_attention, choices= lambda: ["Yes","No"]),
-    AxisOption("ToMe merge cross attention", str, apply_token_merging_cross_attention, choices= lambda: ["Yes","No"]),
-    AxisOption("ToMe merge mlp", str, apply_token_merging_mlp, choices= lambda: ["Yes","No"]),
-    AxisOption("ToMe maximum down sampling", int, apply_token_merging_maximum_down_sampling, choices= lambda: ["1","2","4","8"]),
-    AxisOption("ToMe Stride - X", int, apply_token_merging_stride_x, choices= lambda: ["2","4","6","8"]),
-    AxisOption("ToMe Stride - Y", int, apply_token_merging_stride_y, choices= lambda: ["2","4","6","8"])
+    AxisOption("ToMe random pertubations",str,apply_token_merging_random, choices = lambda: ["Yes","No"])
 ]
 
 
@@ -392,14 +360,7 @@ class SharedSettingsStackHelper(object):
         self.uni_pc_order = opts.uni_pc_order
         self.token_merging_ratio_hr = opts.token_merging_ratio_hr
         self.token_merging_ratio = opts.token_merging_ratio
-        self.token_merging_hr_only = opts.token_merging_hr_only
         self.token_merging_random = opts.token_merging_random
-        self.token_merging_merge_attention = opts.token_merging_merge_attention
-        self.token_merging_merge_cross_attention = opts.token_merging_merge_cross_attention
-        self.token_merging_merge_mlp = opts.token_merging_merge_mlp
-        self.token_merging_maximum_down_sampling = opts.token_merging_maximum_down_sampling
-        self.token_merging_stride_x = opts.token_merging_stride_x
-        self.token_merging_stride_y = opts.token_merging_stride_y
 
     def __exit__(self, exc_type, exc_value, tb):
         #Restore overriden settings after plot generation.
@@ -412,14 +373,7 @@ class SharedSettingsStackHelper(object):
 
         opts.data["token_merging_ratio_hr"] = self.token_merging_ratio_hr
         opts.data["token_merging_ratio"] = self.token_merging_ratio
-        opts.data["token_merging_hr_only"] = self.token_merging_hr_only
         opts.data["token_merging_random"] = self.token_merging_random
-        opts.data["token_merging_merge_attention"] = self.token_merging_merge_attention
-        opts.data["token_merging_merge_cross_attention"] = self.token_merging_merge_cross_attention
-        opts.data["token_merging_merge_mlp"] = self.token_merging_merge_mlp
-        opts.data["token_merging_maximum_down_sampling"] = self.token_merging_maximum_down_sampling
-        opts.data["token_merging_stride_x"] = self.token_merging_stride_x
-        opts.data["token_merging_stride_y"] = self.token_merging_stride_y
 
 re_range = re.compile(r"\s*([+-]?\s*\d+)\s*-\s*([+-]?\s*\d+)(?:\s*\(([+-]\d+)\s*\))?\s*")
 re_range_float = re.compile(r"\s*([+-]?\s*\d+(?:.\d*)?)\s*-\s*([+-]?\s*\d+(?:.\d*)?)(?:\s*\(([+-]\d+(?:.\d*)?)\s*\))?\s*")

From b075d3c8fdf6dece1c66d901a04a651509b2e7fd Mon Sep 17 00:00:00 2001
From: Disty0 <semihgulec2005@gmail.com>
Date: Sun, 30 Apr 2023 15:13:56 +0300
Subject: [PATCH 49/69] Intel ARC Support

---
 modules/cmd_args.py                           |  1 +
 modules/codeformer_model.py                   |  6 +-
 modules/devices.py                            | 33 ++++++++---
 modules/memmon.py                             | 55 ++++++++++++++-----
 modules/processing.py                         | 12 +++-
 modules/sd_hijack_optimizations.py            | 55 ++++++++++++++-----
 modules/sd_hijack_unet.py                     |  3 +-
 modules/sd_models.py                          |  1 -
 modules/shared.py                             |  4 +-
 .../textual_inversion/textual_inversion.py    |  6 +-
 setup.py                                      | 18 +++++-
 11 files changed, 153 insertions(+), 41 deletions(-)

diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index 9b8438800..fdc9ab240 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -23,6 +23,7 @@ parser.add_argument("--allow-code", action='store_true', help="Allow custom scri
 parser.add_argument("--share", action='store_true', help="Enable to make the UI accessible through Gradio site")
 parser.add_argument("--enable-insecure", action='store_true', help="Enable extensions tab regardless of other options")
 parser.add_argument("--use-cpu", nargs='+', help="Force use CPU for specified modules", default=[], type=str.lower)
+parser.add_argument("--use-ipex", action='store_true', help="Force use Intel OneAPI XPU backend")
 parser.add_argument("--listen", action='store_true', help="Launch web server using public IP address")
 parser.add_argument("--port", type=int, help="Launch web server with given server port", default=None)
 parser.add_argument("--hide-ui-dir-config", action='store_true', help="Hide directory configuration from UI", default=False)
diff --git a/modules/codeformer_model.py b/modules/codeformer_model.py
index cbe06ec1e..5217f69db 100644
--- a/modules/codeformer_model.py
+++ b/modules/codeformer_model.py
@@ -103,7 +103,11 @@ def setup_model(dirname):
                             output = self.net(cropped_face_t, w=w if w is not None else shared.opts.code_former_weight, adain=True)[0]
                             restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1))
                         del output
-                        torch.cuda.empty_cache()
+                        from modules import shared
+                        if shared.cmd_opts.use_ipex:
+                            torch.xpu.empty_cache()
+                        else:
+                            torch.cuda.empty_cache()
                     except Exception as error:
                         print(f'\tFailed inference for CodeFormer: {error}', file=sys.stderr)
                         restored_face = tensor2img(cropped_face_t, rgb2bgr=True, min_max=(-1, 1))
diff --git a/modules/devices.py b/modules/devices.py
index e317d91f4..3606597d3 100644
--- a/modules/devices.py
+++ b/modules/devices.py
@@ -22,9 +22,13 @@ def extract_device_id(args, name):
 
 def get_cuda_device_string():
     from modules import shared
-    if shared.cmd_opts.device_id is not None:
-        return f"cuda:{shared.cmd_opts.device_id}"
-    return "cuda"
+    if shared.cmd_opts.use_ipex:
+        return "xpu"
+    else:
+        from modules import shared
+        if shared.cmd_opts.device_id is not None:
+            return f"cuda:{shared.cmd_opts.device_id}"
+        return "cuda"
 
 
 def get_dml_device_string():
@@ -35,7 +39,10 @@ def get_dml_device_string():
 
 
 def get_optimal_device_name():
-    if torch.cuda.is_available():
+    from modules import shared
+    if shared.cmd_opts.use_ipex:
+        return "xpu"
+    elif torch.cuda.is_available():
         return get_cuda_device_string()
     if has_mps():
         return "mps"
@@ -61,7 +68,11 @@ def get_device_for(task):
 
 
 def torch_gc():
-    if torch.cuda.is_available():
+    from modules import shared
+    if shared.cmd_opts.use_ipex:
+        with torch.xpu.device("xpu"):
+            torch.xpu.empty_cache()
+    elif torch.cuda.is_available():
         with torch.cuda.device(get_cuda_device_string()):
             torch.cuda.empty_cache()
             torch.cuda.ipc_collect()
@@ -137,11 +148,19 @@ def autocast(disable=False):
         return contextlib.nullcontext()
     if dtype == torch.float32 or shared.cmd_opts.precision == "Full":
         return contextlib.nullcontext()
-    return torch.autocast("cuda")
+    from modules import shared
+    if shared.cmd_opts.use_ipex:
+        return torch.xpu.amp.autocast(enabled=True, dtype=dtype, cache_enabled=False)
+    else:
+        return torch.autocast("cuda")
 
 
 def without_autocast(disable=False):
-    return torch.autocast("cuda", enabled=False) if torch.is_autocast_enabled() and not disable else contextlib.nullcontext()
+    from modules import shared
+    if shared.cmd_opts.use_ipex:
+        return torch.autocast("xpu", enabled=False) if torch.is_autocast_enabled() and not disable else contextlib.nullcontext()
+    else:
+        return torch.autocast("cuda", enabled=False) if torch.is_autocast_enabled() and not disable else contextlib.nullcontext()
 
 
 class NansException(Exception):
diff --git a/modules/memmon.py b/modules/memmon.py
index 9b013e6b4..4ceb29a37 100644
--- a/modules/memmon.py
+++ b/modules/memmon.py
@@ -19,26 +19,44 @@ class MemUsageMonitor(threading.Thread):
         self.daemon = True
         self.run_flag = threading.Event()
         self.data = defaultdict(int)
-        if not torch.cuda.is_available():
+        from modules import shared
+        if not torch.cuda.is_available() or not shared.cmd_opts.use_ipex:
             self.disabled = True
         else:
-            try:
-                self.cuda_mem_get_info()
-                torch.cuda.memory_stats(self.device)
-            except Exception as e:  # AMD or whatever
-                print(f"Torch exception: {e}")
-                self.disabled = True
+            if shared.cmd_opts.use_ipex:
+                try:
+                    self.cuda_mem_get_info()
+                    torch.cuda.memory_stats("xpu")
+                except Exception as e:  # AMD or whatever
+                    print(f"Torch exception: {e}")
+                    self.disabled = True
+
+            else:
+                try:
+                    self.cuda_mem_get_info()
+                    torch.cuda.memory_stats(self.device)
+                except Exception as e:  # AMD or whatever
+                    print(f"Torch exception: {e}")
+                    self.disabled = True
 
     def cuda_mem_get_info(self):
-        index = self.device.index if self.device.index is not None else torch.cuda.current_device()
-        return torch.cuda.mem_get_info(index)
+        from modules import shared
+        if shared.cmd_opts.use_ipex:
+            return torch.xpu.mem_get_info("xpu")
+        else:
+            index = self.device.index if self.device.index is not None else torch.cuda.current_device()
+            return torch.cuda.mem_get_info(index)
 
     def run(self):
         if self.disabled:
             return
         while True:
             self.run_flag.wait()
-            torch.cuda.reset_peak_memory_stats()
+            from modules import shared
+            if shared.cmd_opts.use_ipex:
+                torch.xpu.reset_peak_memory_stats()
+            else:
+                torch.cuda.reset_peak_memory_stats()
             self.data.clear()
             if self.opts.memmon_poll_rate <= 0:
                 self.run_flag.clear()
@@ -54,12 +72,19 @@ class MemUsageMonitor(threading.Thread):
         for k, v in self.read().items():
             print(k, -(v // -(1024 ** 2)))
         print(self, 'raw torch memory stats:')
-        tm = torch.cuda.memory_stats(self.device)
+        from modules import shared
+        if shared.cmd_opts.use_ipex:
+            tm = torch.xpu.memory_stats("xpu")
+        else:
+            tm = torch.cuda.memory_stats(self.device)
         for k, v in tm.items():
             if 'bytes' not in k:
                 continue
             print('\t' if 'peak' in k else '', k, -(v // -(1024 ** 2)))
-        print(torch.cuda.memory_summary())
+        if shared.cmd_opts.use_ipex:
+            print(torch.xpu.memory_summary())
+        else:
+            print(torch.cuda.memory_summary())
 
     def monitor(self):
         self.run_flag.set()
@@ -70,7 +95,11 @@ class MemUsageMonitor(threading.Thread):
             self.data["free"] = free
             self.data["total"] = total
 
-            torch_stats = torch.cuda.memory_stats(self.device)
+            from modules import shared
+            if shared.cmd_opts.use_ipex:
+                torch_stats = torch.xpu.memory_stats("xpu")
+            else:
+                torch_stats = torch.cuda.memory_stats(self.device)
             self.data["active"] = torch_stats["active.all.current"]
             self.data["active_peak"] = torch_stats["active_bytes.all.peak"]
             self.data["reserved"] = torch_stats["reserved_bytes.all.current"]
diff --git a/modules/processing.py b/modules/processing.py
index e793f12a3..293a8d606 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -55,7 +55,17 @@ def memory_stats():
     except Exception as e:
         mem.update({ 'ram': e })
     try:
-        if torch.cuda.is_available():
+        from modules import shared
+        if shared.cmd_opts.use_ipex:
+            s = torch.xpu.mem_get_info()
+            gpu = { 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }
+            s = dict(torch.xpu.memory_stats("xpu"))
+            mem.update({
+                'gpu': gpu,
+                'retries': s['num_alloc_retries'],
+                'oom': s['num_ooms']
+            })
+        elif torch.cuda.is_available():
             s = torch.cuda.mem_get_info()
             gpu = { 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }
             s = dict(torch.cuda.memory_stats(shared.device))
diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py
index 12ee9f956..5168b4b7a 100644
--- a/modules/sd_hijack_optimizations.py
+++ b/modules/sd_hijack_optimizations.py
@@ -22,7 +22,15 @@ if shared.opts.cross_attention_optimization == "xFormers":
 
 
 def get_available_vram():
-    if shared.device.type == 'cuda':
+    if shared.cmd_opts.use_ipex:
+        stats = torch.xpu.memory_stats("xpu")
+        mem_active = stats['active_bytes.all.current']
+        mem_reserved = stats['reserved_bytes.all.current']
+        mem_free_xpu, _ = torch.xpu.mem_get_info("xpu")
+        mem_free_torch = mem_reserved - mem_active
+        mem_free_total = mem_free_xpu + mem_free_torch
+        return mem_free_total
+    elif shared.device.type == 'cuda':
         stats = torch.cuda.memory_stats(shared.device)
         mem_active = stats['active_bytes.all.current']
         mem_reserved = stats['reserved_bytes.all.current']
@@ -189,14 +197,24 @@ def einsum_op_tensor_mem(q, k, v, max_tensor_mb):
     return einsum_op_slice_1(q, k, v, max(q.shape[1] // div, 1))
 
 def einsum_op_cuda(q, k, v):
-    stats = torch.cuda.memory_stats(q.device)
-    mem_active = stats['active_bytes.all.current']
-    mem_reserved = stats['reserved_bytes.all.current']
-    mem_free_cuda, _ = torch.cuda.mem_get_info(q.device)
-    mem_free_torch = mem_reserved - mem_active
-    mem_free_total = mem_free_cuda + mem_free_torch
-    # Divide factor of safety as there's copying and fragmentation
-    return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20))
+    if shared.cmd_opts.use_ipex:
+        stats = torch.xpu.memory_stats("xpu")
+        mem_active = stats['active_bytes.all.current']
+        mem_reserved = stats['reserved_bytes.all.current']
+        mem_free_xpu, _ = torch.xpu.mem_get_info("xpu")
+        mem_free_torch = mem_reserved - mem_active
+        mem_free_total = mem_free_xpu + mem_free_torch
+        # Divide factor of safety as there's copying and fragmentation
+        return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20))
+    else:
+        stats = torch.cuda.memory_stats(q.device)
+        mem_active = stats['active_bytes.all.current']
+        mem_reserved = stats['reserved_bytes.all.current']
+        mem_free_cuda, _ = torch.cuda.mem_get_info(q.device)
+        mem_free_torch = mem_reserved - mem_active
+        mem_free_total = mem_free_cuda + mem_free_torch
+        # Divide factor of safety as there's copying and fragmentation
+        return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20))
 
 def einsum_op_dml(q, k, v):
     mem_total, mem_active = torch.dml.memory_stats(q.device)
@@ -204,6 +222,9 @@ def einsum_op_dml(q, k, v):
     return einsum_op_tensor_mem(q, k, v, (mem_reserved - mem_active) if mem_reserved > mem_active else 1)
 
 def einsum_op(q, k, v):
+    if shared.cmd_opts.use_ipex:
+        return einsum_op_cuda(q, k, v)
+
     if q.device.type == 'cuda':
         return einsum_op_cuda(q, k, v)
 
@@ -397,8 +418,12 @@ def scaled_dot_product_attention_forward(self, x, context=None, mask=None):
     return hidden_states
 
 def scaled_dot_product_no_mem_attention_forward(self, x, context=None, mask=None):
-    with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False):
-        return scaled_dot_product_attention_forward(self, x, context, mask)
+    if shared.cmd_opts.use_ipex:
+        with torch.backends.xpu.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False):
+            return scaled_dot_product_attention_forward(self, x, context, mask)
+    else:
+        with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False):
+            return scaled_dot_product_attention_forward(self, x, context, mask)
 
 def cross_attention_attnblock_forward(self, x):
     h_ = x
@@ -502,8 +527,12 @@ def sdp_attnblock_forward(self, x):
     return x + out
 
 def sdp_no_mem_attnblock_forward(self, x):
-    with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False):
-        return sdp_attnblock_forward(self, x)
+    if shared.cmd_opts.use_ipex:
+        with torch.backends.xpu.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False):
+            return sdp_attnblock_forward(self, x)
+    else:
+        with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False):
+            return sdp_attnblock_forward(self, x)
 
 def sub_quad_attnblock_forward(self, x):
     h_ = x
diff --git a/modules/sd_hijack_unet.py b/modules/sd_hijack_unet.py
index 158582632..7ff553ae3 100644
--- a/modules/sd_hijack_unet.py
+++ b/modules/sd_hijack_unet.py
@@ -3,6 +3,7 @@ from packaging import version
 
 from modules import devices
 from modules.sd_hijack_utils import CondFunc
+from modules import shared
 
 
 class TorchHijackForUnet:
@@ -67,7 +68,7 @@ def hijack_ddpm_edit():
 unet_needs_upcast = lambda *args, **kwargs: devices.unet_needs_upcast
 CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.apply_model', apply_model, unet_needs_upcast)
 CondFunc('ldm.modules.diffusionmodules.openaimodel.timestep_embedding', lambda orig_func, timesteps, *args, **kwargs: orig_func(timesteps, *args, **kwargs).to(torch.float32 if timesteps.dtype == torch.int64 else devices.dtype_unet), unet_needs_upcast)
-if version.parse(torch.__version__) <= version.parse("1.13.2") or torch.cuda.is_available():
+if version.parse(torch.__version__) <= version.parse("1.13.2") or torch.cuda.is_available() or shared.cmd_opts.use_ipex:
     CondFunc('ldm.modules.diffusionmodules.util.GroupNorm32.forward', lambda orig_func, self, *args, **kwargs: orig_func(self.float(), *args, **kwargs), unet_needs_upcast)
     CondFunc('ldm.modules.attention.GEGLU.forward', lambda orig_func, self, x: orig_func(self.float(), x.float()).to(devices.dtype_unet), unet_needs_upcast)
     CondFunc('open_clip.transformer.ResidualAttentionBlock.__init__', lambda orig_func, *args, **kwargs: kwargs.update({'act_layer': GELUHijack}) and False or orig_func(*args, **kwargs), lambda _, *args, **kwargs: kwargs.get('act_layer') is None or kwargs['act_layer'] == torch.nn.GELU)
diff --git a/modules/sd_models.py b/modules/sd_models.py
index a2ef7a012..49de37097 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -533,7 +533,6 @@ def unload_model_weights(sd_model=None, _info=None):
         sd_model = None
         gc.collect()
         devices.torch_gc()
-        torch.cuda.empty_cache()
     print(f"Unloaded weights {timer.summary()}")
     return sd_model
 
diff --git a/modules/shared.py b/modules/shared.py
index f3ebbce55..aa2ede29e 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -238,7 +238,7 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), {
     "comma_padding_backtrack": OptionInfo(20, "Increase coherency by padding from the last comma within n tokens when using more than 75 tokens", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }),
     "CLIP_stop_at_last_layers": OptionInfo(1, "Clip skip", gr.Slider, {"minimum": 1, "maximum": 12, "step": 1, "visible": False}),
     "upcast_attn": OptionInfo(False, "Upcast cross attention layer to float32"),
-    "cross_attention_optimization": OptionInfo("Sub-quadratic" if is_device_dml else "Scaled-Dot-Product", "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }),
+    "cross_attention_optimization": OptionInfo("Sub-quadratic" if is_device_dml else "Split attention" if cmd_opts.use_ipex else "Scaled-Dot-Product", "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }),
     "cross_attention_options": OptionInfo([], "Cross-attention advanced options", gr.CheckboxGroup, lambda: {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention']}),
     "sub_quad_q_chunk_size": OptionInfo(512, "Sub-quadratic cross-attention query chunk size for the  layer optimization to use", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8}),
     "sub_quad_kv_chunk_size": OptionInfo(512, "Sub-quadratic cross-attentionkv chunk size for the sub-quadratic cross-attention layer optimization to use", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8}),
@@ -318,7 +318,7 @@ options_templates.update(options_section(('cuda', "CUDA Settings"), {
     "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" else "FP16", "Device precision type", gr.Radio, lambda: {"choices": ["FP32", "FP16", "BF16"]}),
     "no_half": OptionInfo(True if is_device_dml else False, "Use full precision for model (--no-half)", None, None, lambda: print("Warning: Most of DirectML devices do not fully support half mode. Recommend to use full precision to model.") if is_device_dml else None),
     "no_half_vae": OptionInfo(True if is_device_dml else False, "Use full precision for VAE (--no-half-vae)"),
-    "upcast_sampling": OptionInfo(True if sys.platform == "darwin" else False, "Enable upcast sampling. Usually produces similar results to --no-half with better performance while using less memory"),
+    "upcast_sampling": OptionInfo(True if sys.platform == "darwin" or cmd_opts.use_ipex else False, "Enable upcast sampling. Usually produces similar results to --no-half with better performance while using less memory"),
     "disable_nan_check": OptionInfo(True, "Do not check if produced images/latent spaces have NaN values"),
     "rollback_vae": OptionInfo(False, "Attempt to roll back VAE when produced NaN values, requires NaN check (experimental)"),
     "opt_channelslast": OptionInfo(False, "Use channels last as torch memory format "),
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index 2180d7f32..36a1e1e17 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -434,7 +434,11 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
         else:
             print("No saved optimizer exists in checkpoint")
 
-    scaler = torch.cuda.amp.GradScaler()
+    from modules import shared
+    if shared.cmd_opts.use_ipex:
+        scaler = torch.xpu.amp.GradScaler()
+    else:
+        scaler = torch.cuda.amp.GradScaler()
 
     batch_size = ds.batch_size
     gradient_step = ds.gradient_step
diff --git a/setup.py b/setup.py
index cfaa1aaa0..59c200161 100644
--- a/setup.py
+++ b/setup.py
@@ -56,6 +56,7 @@ def setup_logging(clean=False):
 # check if package is installed
 def installed(package, friendly: str = None):
     import pkg_resources
+    from modules import shared
     ok = True
     try:
         if friendly:
@@ -76,6 +77,8 @@ def installed(package, friendly: str = None):
             ok = ok and spec is not None
             if ok:
                 version = pkg_resources.get_distribution(p[0]).version
+                if shared.cmd_opts.use_ipex and p[0] == "pytorch_lightning":
+                    p[1] = "1.8.6"
                 log.debug(f"Package version found: {p[0]} {version}")
                 if len(p) > 1:
                     ok = ok and version == p[1]
@@ -91,6 +94,9 @@ def installed(package, friendly: str = None):
 
 # install package using pip if not already installed
 def install(package, friendly: str = None, ignore: bool = False):
+    from modules import shared
+    if shared.cmd_opts.use_ipex and package == "pytorch_lightning==1.9.4":
+        package = "pytorch_lightning==1.8.6"
     def pip(arg: str):
         arg = arg.replace('>=', '==')
         log.info(f'Installing package: {arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("  ", " ").strip()}')
@@ -188,6 +194,7 @@ def check_python():
 
 # check torch version
 def check_torch():
+    from modules import shared
     if shutil.which('nvidia-smi') is not None or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe')):
         log.info('nVidia toolkit detected')
         torch_command = os.environ.get('TORCH_COMMAND', 'torch torchaudio torchvision --index-url https://download.pytorch.org/whl/cu118')
@@ -197,6 +204,11 @@ def check_torch():
         os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', '10.3.0')
         torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.4.2')
         xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
+    elif shutil.which('sycl-ls') is not None or os.path.exists('/opt/intel/oneapi'):
+        shared.cmd_opts.use_ipex = True
+        log.info('Intel toolkit detected')
+        torch_command = os.environ.get('TORCH_COMMAND', 'torch==1.13.0a0+git6c9b55e torchvision==0.14.1a0 intel_extension_for_pytorch==1.13.120+xpu --index-url https://developer.intel.com/ipex-whl-stable-xpu')
+        xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
     else:
         machine = platform.machine()
         if 'arm' not in machine and 'aarch' not in machine and not args.nodirectml: # torch-directml is available on AMD64
@@ -212,7 +224,11 @@ def check_torch():
     try:
         import torch
         log.info(f'Torch {torch.__version__}')
-        if torch.cuda.is_available():
+        if shared.cmd_opts.use_ipex:
+                import intel_extension_for_pytorch as ipex
+                log.info(f'Torch backend: Intel OneAPI {torch.__version__}')
+                log.info(f'Torch detected GPU: {torch.xpu.get_device_name("xpu")} VRAM {round(torch.xpu.get_device_properties("xpu").total_memory / 1024 / 1024)}')
+        elif torch.cuda.is_available():
             if torch.version.cuda:
                 log.info(f'Torch backend: nVidia CUDA {torch.version.cuda} cuDNN {torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else "N/A"}')
             elif torch.version.hip:

From 5c76087b9d4e90edb7f79631c3ef1cc3e627327d Mon Sep 17 00:00:00 2001
From: Disty0 <semihgulec2005@gmail.com>
Date: Sun, 30 Apr 2023 15:30:15 +0300
Subject: [PATCH 50/69] Revert force cross_attention_optimization

---
 modules/shared.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/shared.py b/modules/shared.py
index aa2ede29e..12cb05b05 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -238,7 +238,7 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), {
     "comma_padding_backtrack": OptionInfo(20, "Increase coherency by padding from the last comma within n tokens when using more than 75 tokens", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }),
     "CLIP_stop_at_last_layers": OptionInfo(1, "Clip skip", gr.Slider, {"minimum": 1, "maximum": 12, "step": 1, "visible": False}),
     "upcast_attn": OptionInfo(False, "Upcast cross attention layer to float32"),
-    "cross_attention_optimization": OptionInfo("Sub-quadratic" if is_device_dml else "Split attention" if cmd_opts.use_ipex else "Scaled-Dot-Product", "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }),
+    "cross_attention_optimization": OptionInfo("Sub-quadratic" if is_device_dml else "Scaled-Dot-Product", "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }),
     "cross_attention_options": OptionInfo([], "Cross-attention advanced options", gr.CheckboxGroup, lambda: {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention']}),
     "sub_quad_q_chunk_size": OptionInfo(512, "Sub-quadratic cross-attention query chunk size for the  layer optimization to use", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8}),
     "sub_quad_kv_chunk_size": OptionInfo(512, "Sub-quadratic cross-attentionkv chunk size for the sub-quadratic cross-attention layer optimization to use", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8}),

From b23b6a6e2c002eaf91693ef7997158bf8c279cdf Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sun, 30 Apr 2023 08:55:44 -0400
Subject: [PATCH 51/69] update ti folders

---
 extensions-builtin/multidiffusion-upscaler-for-automatic1111 | 2 +-
 extensions-builtin/sd-webui-controlnet                       | 2 +-
 modules/lora                                                 | 2 +-
 modules/shared.py                                            | 4 ++--
 modules/textual_inversion/textual_inversion.py               | 5 +++--
 5 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/extensions-builtin/multidiffusion-upscaler-for-automatic1111 b/extensions-builtin/multidiffusion-upscaler-for-automatic1111
index 6931b89cb..860f8a405 160000
--- a/extensions-builtin/multidiffusion-upscaler-for-automatic1111
+++ b/extensions-builtin/multidiffusion-upscaler-for-automatic1111
@@ -1 +1 @@
-Subproject commit 6931b89cb4507c7dc8fa81ac36c2c19d0691c44e
+Subproject commit 860f8a405193bcd992e21d82e43fa18137bc4923
diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index 4d4b1f8c0..09d1fcbf4 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit 4d4b1f8c00a0355d1517465ac3c0e801d5a2d194
+Subproject commit 09d1fcbf4dc715bca7547496f850801f95f732a6
diff --git a/modules/lora b/modules/lora
index d52c524fc..bc803e01c 160000
--- a/modules/lora
+++ b/modules/lora
@@ -1 +1 @@
-Subproject commit d52c524fc2942c053cf37c648188502a3a26df1b
+Subproject commit bc803e01c7028471efc8db5bc9aa183fde06080c
diff --git a/modules/shared.py b/modules/shared.py
index f3ebbce55..29a1786da 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -252,8 +252,6 @@ options_templates.update(options_section(('system-paths', "System Paths"), {
     "ckpt_dir": OptionInfo(os.path.join(paths.models_path, 'Stable-diffusion'), "Path to directory with stable diffusion checkpoints"),
     "vae_dir": OptionInfo(os.path.join(paths.models_path, 'VAE'), "Path to directory with VAE files"),
     "embeddings_dir": OptionInfo(os.path.join(paths.models_path, 'embeddings'), "Embeddings directory for textual inversion"),
-    "embeddings_templates_dir": OptionInfo(os.path.join(paths.script_path, 'train/templates'), "Embeddings train templates directory"),
-    "embeddings_train_log": OptionInfo(os.path.join(paths.script_path, 'train.csv'), "Embeddings train log file"),
     "hypernetwork_dir": OptionInfo(os.path.join(paths.models_path, 'hypernetworks'), "Hypernetwork directory"),
     "codeformer_models_path": OptionInfo(os.path.join(paths.models_path, 'Codeformer'), "Path to directory with codeformer model file(s)."),
     "gfpgan_models_path": OptionInfo(os.path.join(paths.models_path, 'GFPGAN'), "Path to directory with GFPGAN model file(s)"),
@@ -351,6 +349,8 @@ options_templates.update(options_section(('training', "Training"), {
     "save_training_settings_to_txt": OptionInfo(True, "Save textual inversion and hypernet settings to a text file whenever training starts."),
     "dataset_filename_word_regex": OptionInfo("", "Filename word regex"),
     "dataset_filename_join_string": OptionInfo(" ", "Filename join string"),
+    "embeddings_templates_dir": OptionInfo(os.path.join(paths.script_path, 'train', 'templates'), "Embeddings train templates directory"),
+    "embeddings_train_log": OptionInfo(os.path.join(paths.script_path, 'train', 'log', 'train.csv'), "Embeddings train log file"),
     "training_image_repeats_per_epoch": OptionInfo(1, "Number of repeats for a single input image per epoch; used only for displaying epoch number", gr.Number, {"precision": 0}),
     "training_write_csv_every": OptionInfo(0, "Save an csv containing the loss to log directory every N steps, 0 to disable"),
     "training_enable_tensorboard": OptionInfo(False, "Enable tensorboard logging."),
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index 2180d7f32..cbacc2ce2 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -174,7 +174,7 @@ class EmbeddingDatabase:
             if len(emb.shape) == 1:
                 emb = emb.unsqueeze(0)
         else:
-            raise Exception(f"Couldn't identify {filename} as neither textual inversion embedding nor diffuser concept.")
+            raise RuntimeError(f"Couldn't identify {filename} as neither textual inversion embedding nor diffuser concept.")
 
         vec = emb.detach().to(devices.device, dtype=torch.float32)
         embedding = Embedding(vec, name)
@@ -347,7 +347,8 @@ def validate_train_inputs(model_name, learn_rate, batch_size, gradient_step, dat
         assert log_directory, "Log directory is empty"
 
 
-def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, varsize, steps, clip_grad_mode, clip_grad_value, shuffle_tags, tag_drop_out, latent_sampling_method, use_weight, create_image_every, save_embedding_every, template_filename, save_image_with_stored_embedding, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
+def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, varsize, steps, clip_grad_mode, clip_grad_value, shuffle_tags, tag_drop_out, latent_sampling_method, use_weight, create_image_every, save_embedding_every, template_filename, save_image_with_stored_embedding, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height): # pylint: disable=unused_argument
+
     save_embedding_every = save_embedding_every or 0
     create_image_every = create_image_every or 0
     template_file = textual_inversion_templates.get(template_filename, None)

From a720a670e826715790d095d26a3829126f6b7811 Mon Sep 17 00:00:00 2001
From: Disty0 <semihgulec2005@gmail.com>
Date: Sun, 30 Apr 2023 16:01:17 +0300
Subject: [PATCH 52/69] More patches and less import shared

---
 cli/modules/bench.py                          |  7 ++-
 cli/modules/interrogate-offline.py            | 18 ++++++-
 cli/modules/lora-extract.py                   | 16 +++++-
 cli/modules/lora-latents.py                   | 11 ++++-
 cli/modules/util.py                           | 21 +++++++-
 cli/random/dynamotest.py                      | 49 ++++++++++++++-----
 cli/train-lora.py                             | 13 ++++-
 cli/train/latents.py                          | 11 ++++-
 .../multidiffusion-upscaler-for-automatic1111 |  2 +-
 extensions-builtin/sd-webui-controlnet        |  2 +-
 modules/devices.py                            | 17 +++----
 modules/lora                                  |  2 +-
 12 files changed, 132 insertions(+), 37 deletions(-)

diff --git a/cli/modules/bench.py b/cli/modules/bench.py
index 094b73f63..18791bfc9 100755
--- a/cli/modules/bench.py
+++ b/cli/modules/bench.py
@@ -10,7 +10,7 @@ import time
 from PIL import Image
 import sdapi
 from util import Map, log
-
+from modules import shared
 
 options = Map({
     'restore_faces': False,
@@ -56,7 +56,10 @@ async def txt2img():
 def memstats():
     mem = sdapi.getsync('/sdapi/v1/memory')
     cpu = mem.get('ram', 'unavailable')
-    gpu = mem.get('cuda', 'unavailable')
+    if shared.cmd_opts.use_ipex:
+        gpu = mem.get('xpu', 'unavailable')
+    else:
+        gpu = mem.get('cuda', 'unavailable')
     if 'active' in gpu:
         gpu['session'] = gpu.pop('active')
     if 'reserved' in gpu:
diff --git a/cli/modules/interrogate-offline.py b/cli/modules/interrogate-offline.py
index 6d9ae56fa..c2623cda6 100755
--- a/cli/modules/interrogate-offline.py
+++ b/cli/modules/interrogate-offline.py
@@ -6,6 +6,12 @@ import json
 import time
 import argparse
 import torch
+from modules import shared
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    if shared.cmd_opts.use_ipex:
+        print("Failed to import IPEX")
 import filetype
 from PIL import Image
 import transformers
@@ -19,7 +25,10 @@ model = None
 processor = None
 extractor = None
 dtype = torch.float32
-device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+if shared.cmd_opts.use_ipex:
+    device = torch.device('xpu')
+else:
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
 options = Map({
     'input': '',
@@ -129,7 +138,12 @@ def unload_model():
         del extractor
         extractor = None
     gc.collect()
-    if torch.cuda.is_available():
+    if shared.cmd_opts.use_ipex:
+        with torch.no_grad():
+            torch.xpu.empty_cache()
+        with torch.xpu.device('xpu'):
+            torch.xpu.empty_cache()
+    elif torch.cuda.is_available():
         with torch.no_grad():
             torch.cuda.empty_cache()
         with torch.cuda.device('cuda'):
diff --git a/cli/modules/lora-extract.py b/cli/modules/lora-extract.py
index 102728308..9a781789a 100755
--- a/cli/modules/lora-extract.py
+++ b/cli/modules/lora-extract.py
@@ -10,6 +10,12 @@ import sys
 import time
 import argparse
 import torch
+from modules import shared
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    if shared.cmd_opts.use_ipex:
+        print("Failed to import IPEX")
 import transformers
 from tqdm import tqdm
 from util import log
@@ -20,7 +26,10 @@ import networks.lora as lora
 
 
 def svd(args): # pylint: disable=redefined-outer-name
-    device = 'cuda' if torch.cuda.is_available() and args.device == 'cuda' else 'cpu'
+    if shared.cmd_opts.use_ipex:
+        device = torch.device('xpu')
+    else:
+        device = 'cuda' if torch.cuda.is_available() and args.device == 'cuda' else 'cpu'
     transformers.logging.set_verbosity_error()
     CLAMP_QUANTILE = 0.99
     MIN_DIFF = 1e-6
@@ -38,7 +47,10 @@ def svd(args): # pylint: disable=redefined-outer-name
     log.info({ 'loading model': args.tuned })
     text_encoder_t, _, unet_t = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.tuned)
     with torch.no_grad():
-        torch.cuda.empty_cache()
+        if shared.cmd_opts.use_ipex:
+            torch.xpu.empty_cache()
+        else:
+            torch.cuda.empty_cache()
     # create LoRA network to extract weights: Use dim (rank) as alpha
     lora_network_o = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_o, unet_o)
     lora_network_t = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_t, unet_t)
diff --git a/cli/modules/lora-latents.py b/cli/modules/lora-latents.py
index d556d596b..7e701df12 100755
--- a/cli/modules/lora-latents.py
+++ b/cli/modules/lora-latents.py
@@ -10,6 +10,12 @@ import warnings
 import cv2
 import numpy as np
 import torch
+from modules import shared
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    if shared.cmd_opts.use_ipex:
+        print("Failed to import IPEX")
 from PIL import Image
 from torchvision import transforms
 from tqdm import tqdm
@@ -20,7 +26,10 @@ import library.model_util as model_util
 import library.train_util as train_util
 
 warnings.filterwarnings('ignore')
-device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+if shared.cmd_opts.use_ipex:
+    device = torch.device('xpu')
+else:
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 options = Map({
   'batch': 1,
   'input': '',
diff --git a/cli/modules/util.py b/cli/modules/util.py
index 479b77233..d48887961 100755
--- a/cli/modules/util.py
+++ b/cli/modules/util.py
@@ -44,7 +44,26 @@ def get_memory():
         mem.update({ 'ram': e })
     try:
         import torch
-        if torch.cuda.is_available():
+        from modules import shared
+        if shared.cmd_opts.use_ipex:
+            import intel_extension_for_pytorch as ipex
+            s = torch.xpu.mem_get_info()
+            gpu = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }
+            s = dict(torch.xpu.memory_stats('xpu'))
+            allocated = { 'current': gb(s['allocated_bytes.all.current']), 'peak': gb(s['allocated_bytes.all.peak']) }
+            reserved = { 'current': gb(s['reserved_bytes.all.current']), 'peak': gb(s['reserved_bytes.all.peak']) }
+            active = { 'current': gb(s['active_bytes.all.current']), 'peak': gb(s['active_bytes.all.peak']) }
+            inactive = { 'current': gb(s['inactive_split_bytes.all.current']), 'peak': gb(s['inactive_split_bytes.all.peak']) }
+            warnings = { 'retries': s['num_alloc_retries'], 'oom': s['num_ooms'] }
+            mem.update({
+                'gpu': gpu,
+                'gpu-active': active,
+                'gpu-allocated': allocated,
+                'gpu-reserved': reserved,
+                'gpu-inactive': inactive,
+                'events': warnings,
+            })
+        elif torch.cuda.is_available():
             s = torch.cuda.mem_get_info()
             gpu = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }
             s = dict(torch.cuda.memory_stats('cuda'))
diff --git a/cli/random/dynamotest.py b/cli/random/dynamotest.py
index 82b1143c6..556ae96a8 100755
--- a/cli/random/dynamotest.py
+++ b/cli/random/dynamotest.py
@@ -7,9 +7,14 @@ import warnings
 
 import numpy as np
 import torch
+from modules import shared
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    if shared.cmd_opts.use_ipex:
+        print("Failed to import IPEX")
 from torchvision.models import resnet18
 
-
 print('torch:', torch.__version__)
 try:
     import torch._dynamo as dynamo # must be imported explicitly or namespace is not found
@@ -24,24 +29,42 @@ warnings.filterwarnings('ignore', category=UserWarning) # disable those for now
 
 
 def timed(fn): # returns the result of running `fn()` and the time it took for `fn()` to run in ms using CUDA events
-    start = torch.cuda.Event(enable_timing=True)
-    end = torch.cuda.Event(enable_timing=True)
-    start.record()
-    result = fn()
-    end.record()
-    torch.cuda.synchronize()
-    return result, start.elapsed_time(end)
+    if shared.cmd_opts.use_ipex:
+        start = torch.xpu.Event(enable_timing=True)
+        end = torch.xpu.Event(enable_timing=True)
+        start.record()
+        result = fn()
+        end.record()
+        torch.xpu.synchronize()
+        return result, start.elapsed_time(end)
+    else:
+        start = torch.cuda.Event(enable_timing=True)
+        end = torch.cuda.Event(enable_timing=True)
+        start.record()
+        result = fn()
+        end.record()
+        torch.cuda.synchronize()
+        return result, start.elapsed_time(end)
 
 
 def generate_data(b):
-    return (
-        torch.randn(b, 3, 128, 128).to(torch.float32).cuda(),
-        torch.randint(1000, (b,)).cuda(),
-    )
+    if shared.cmd_opts.use_ipex:
+        return (
+            torch.randn(b, 3, 128, 128).to(torch.float32).xpu(),
+            torch.randint(1000, (b,)).xpu(),
+        )
+    else:
+        return (
+            torch.randn(b, 3, 128, 128).to(torch.float32).cuda(),
+            torch.randint(1000, (b,)).cuda(),
+        )
 
 
 def init_model():
-    return resnet18().to(torch.float32).cuda()
+    if shared.cmd_opts.use_ipex:
+        return resnet18().to(torch.float32).xpu()
+    else:
+        return resnet18().to(torch.float32).cuda()
 
 
 def eval(mod, inp):
diff --git a/cli/train-lora.py b/cli/train-lora.py
index 6f82063a6..f1e295a35 100755
--- a/cli/train-lora.py
+++ b/cli/train-lora.py
@@ -23,6 +23,12 @@ import shutil
 import argparse
 import tempfile
 import torch
+from modules import shared
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    if shared.cmd_opts.use_ipex:
+        print("Failed to import IPEX")
 import logging
 import importlib
 import transformers
@@ -117,7 +123,12 @@ options = Map({
 
 def mem_stats():
     gc.collect()
-    if torch.cuda.is_available():
+    if shared.cmd_opts.use_ipex:
+        with torch.no_grad():
+            torch.xpu.empty_cache()
+        with torch.xpu.device('xpu'):
+            torch.cuda.empty_cache()
+    elif torch.cuda.is_available():
         with torch.no_grad():
             torch.cuda.empty_cache()
         with torch.cuda.device('cuda'):
diff --git a/cli/train/latents.py b/cli/train/latents.py
index 94249b18a..715f92aba 100755
--- a/cli/train/latents.py
+++ b/cli/train/latents.py
@@ -10,6 +10,12 @@ import warnings
 import cv2
 import numpy as np
 import torch
+from modules import shared
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    if shared.cmd_opts.use_ipex:
+        print("Failed to import IPEX")
 from PIL import Image
 from torchvision import transforms
 from tqdm import tqdm
@@ -28,7 +34,10 @@ import library.model_util as model_util
 import library.train_util as train_util
 
 warnings.filterwarnings('ignore')
-device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+if shared.cmd_opts.use_ipex:
+    device = torch.device('xpu')
+else:
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 options = Map({
   'batch': 1,
   'input': '',
diff --git a/extensions-builtin/multidiffusion-upscaler-for-automatic1111 b/extensions-builtin/multidiffusion-upscaler-for-automatic1111
index 6931b89cb..860f8a405 160000
--- a/extensions-builtin/multidiffusion-upscaler-for-automatic1111
+++ b/extensions-builtin/multidiffusion-upscaler-for-automatic1111
@@ -1 +1 @@
-Subproject commit 6931b89cb4507c7dc8fa81ac36c2c19d0691c44e
+Subproject commit 860f8a405193bcd992e21d82e43fa18137bc4923
diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index 4d4b1f8c0..09d1fcbf4 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit 4d4b1f8c00a0355d1517465ac3c0e801d5a2d194
+Subproject commit 09d1fcbf4dc715bca7547496f850801f95f732a6
diff --git a/modules/devices.py b/modules/devices.py
index 3606597d3..1ab082f33 100644
--- a/modules/devices.py
+++ b/modules/devices.py
@@ -1,6 +1,12 @@
 import sys
 import contextlib
 import torch
+from modules import shared
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    if shared.cmd_opts.use_ipex:
+        print("Failed to import IPEX")
 
 if sys.platform == "darwin":
     from modules import mac_specific
@@ -21,25 +27,21 @@ def extract_device_id(args, name):
 
 
 def get_cuda_device_string():
-    from modules import shared
     if shared.cmd_opts.use_ipex:
         return "xpu"
     else:
-        from modules import shared
         if shared.cmd_opts.device_id is not None:
             return f"cuda:{shared.cmd_opts.device_id}"
         return "cuda"
 
 
 def get_dml_device_string():
-    from modules import shared
     if shared.cmd_opts.device_id is not None:
         return f"privateuseone:{shared.cmd_opts.device_id}"
     return "privateuseone:0"
 
 
 def get_optimal_device_name():
-    from modules import shared
     if shared.cmd_opts.use_ipex:
         return "xpu"
     elif torch.cuda.is_available():
@@ -61,14 +63,12 @@ def get_optimal_device():
 
 
 def get_device_for(task):
-    from modules import shared
     if task in shared.cmd_opts.use_cpu:
         return cpu
     return get_optimal_device()
 
 
 def torch_gc():
-    from modules import shared
     if shared.cmd_opts.use_ipex:
         with torch.xpu.device("xpu"):
             torch.xpu.empty_cache()
@@ -79,7 +79,6 @@ def torch_gc():
 
 
 def set_cuda_params():
-    from modules import shared
     if torch.cuda.is_available():
         try:
             torch.backends.cuda.matmul.allow_tf32 = shared.opts.cuda_allow_tf32
@@ -143,12 +142,10 @@ def randn_without_seed(shape):
 
 
 def autocast(disable=False):
-    from modules import shared
     if disable:
         return contextlib.nullcontext()
     if dtype == torch.float32 or shared.cmd_opts.precision == "Full":
         return contextlib.nullcontext()
-    from modules import shared
     if shared.cmd_opts.use_ipex:
         return torch.xpu.amp.autocast(enabled=True, dtype=dtype, cache_enabled=False)
     else:
@@ -156,7 +153,6 @@ def autocast(disable=False):
 
 
 def without_autocast(disable=False):
-    from modules import shared
     if shared.cmd_opts.use_ipex:
         return torch.autocast("xpu", enabled=False) if torch.is_autocast_enabled() and not disable else contextlib.nullcontext()
     else:
@@ -168,7 +164,6 @@ class NansException(Exception):
 
 
 def test_for_nans(x, where):
-    from modules import shared
     if shared.opts.disable_nan_check:
         return
     if not torch.all(torch.isnan(x)).item():
diff --git a/modules/lora b/modules/lora
index d52c524fc..bc803e01c 160000
--- a/modules/lora
+++ b/modules/lora
@@ -1 +1 @@
-Subproject commit d52c524fc2942c053cf37c648188502a3a26df1b
+Subproject commit bc803e01c7028471efc8db5bc9aa183fde06080c

From 14055afb9b7d31b65af2da2b8d8e7483241b028d Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sun, 30 Apr 2023 09:20:50 -0400
Subject: [PATCH 53/69] update logging

---
 modules/textual_inversion/logging.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/modules/textual_inversion/logging.py b/modules/textual_inversion/logging.py
index edef48433..d0e52beb7 100644
--- a/modules/textual_inversion/logging.py
+++ b/modules/textual_inversion/logging.py
@@ -2,7 +2,7 @@ import datetime
 import json
 import os
 
-saved_params_shared = {"model_name", "model_hash", "initial_step", "num_of_dataset_images", "learn_rate", "batch_size", "clip_grad_mode", "clip_grad_value", "gradient_step", "data_root", "log_directory", "training_width", "training_height", "steps", "create_image_every", "template_file", "gradient_step", "latent_sampling_method"}
+saved_params_shared = {"model_name", "model_hash", "initial_step", "num_of_dataset_images", "learn_rate", "batch_size", "clip_grad_mode", "clip_grad_value", "gradient_step", "data_root", "log_directory", "training_width", "training_height", "steps", "create_image_every", "template_file", "latent_sampling_method"}
 saved_params_ti = {"embedding_name", "num_vectors_per_token", "save_embedding_every", "save_image_with_stored_embedding"}
 saved_params_hypernet = {"hypernetwork_name", "layer_structure", "activation_func", "weight_init", "add_layer_norm", "use_dropout", "save_hypernetwork_every"}
 saved_params_all = saved_params_shared | saved_params_ti | saved_params_hypernet
@@ -12,13 +12,12 @@ saved_params_previews = {"preview_prompt", "preview_negative_prompt", "preview_s
 def save_settings_to_file(log_directory, all_params):
     now = datetime.datetime.now()
     params = {"datetime": now.strftime("%Y-%m-%d %H:%M:%S")}
-
     keys = saved_params_all
     if all_params.get('preview_from_txt2img'):
         keys = keys | saved_params_previews
-
     params.update({k: v for k, v in all_params.items() if k in keys})
-
-    filename = f'settings.json'
-    with open(os.path.join(log_directory, filename), "w") as file:
+    filename = 'settings.json'
+    fn = os.path.join(log_directory, filename)
+    with open(os.path.join(log_directory, filename), "w", encoding='utf-8') as file:
+        print(f'Training settings file: {fn}')
         json.dump(params, file, indent=2)

From 917ecad43c4da4f3aad54048284ad0ee44a04634 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sun, 30 Apr 2023 09:31:38 -0400
Subject: [PATCH 54/69] add dynamo options

---
 extensions-builtin/sd-webui-controlnet | 2 +-
 modules/sd_hijack.py                   | 3 ++-
 modules/shared.py                      | 2 ++
 wiki                                   | 2 +-
 4 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index 09d1fcbf4..d2da774a4 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit 09d1fcbf4dc715bca7547496f850801f95f732a6
+Subproject commit d2da774a40ff9c3770e21f71fb516403022fc3f6
diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index f817b7afd..47640377c 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -178,7 +178,8 @@ class StableDiffusionModelHijack:
         if opts.cuda_compile and opts.cuda_compile_mode != 'none':
             try:
                 import torch._dynamo as dynamo # pylint: disable=unused-import
-                torch._dynamo.config.verbose = True # pylint: disable=protected-access
+                torch._dynamo.config.verbose = opts.cuda_compile_verbose # pylint: disable=protected-access
+                torch._dynamo.config.suppress_errors = opts.cuda_compile_errors # pylint: disable=protected-access
                 torch.backends.cudnn.benchmark = True
                 m.model = torch.compile(m.model, mode="default", backend=opts.cuda_compile_mode, fullgraph=False, dynamic=False)
                 print("Model compile enabled:", opts.cuda_compile_mode)
diff --git a/modules/shared.py b/modules/shared.py
index af65368da..288804fea 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -325,6 +325,8 @@ options_templates.update(options_section(('cuda', "CUDA Settings"), {
     "cuda_allow_tf16_reduced": OptionInfo(True, "Allow TF16 reduced precision math ops"),
     "cuda_compile": OptionInfo(False, "Enable model compile (experimental)"),
     "cuda_compile_mode": OptionInfo("none", "Model compile mode (experimental)", gr.Radio, lambda: {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser']}),
+    "cuda_compile_verbose": OptionInfo(True, "Compile verbose mode"),
+    "cuda_compile_errors": OptionInfo(True, "Compile suppress errors"),
 }))
 
 options_templates.update(options_section(('upscaling', "Upscaling"), {
diff --git a/wiki b/wiki
index 6cd8fde16..4cbdffaa9 160000
--- a/wiki
+++ b/wiki
@@ -1 +1 @@
-Subproject commit 6cd8fde165190057c0849fa6f8dbb183f717b176
+Subproject commit 4cbdffaa95978d0a46758eac4a3fbe689eb4cdcd

From 682330b172b35a5f91a1bdcb74a2d84a6fc978ec Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sun, 30 Apr 2023 10:54:59 -0400
Subject: [PATCH 55/69] new command line parser

---
 TODO.md              |   3 +-
 modules/cmd_args.py  | 141 ++++++++++++++++++++++---------------------
 modules/sd_hijack.py |  14 +++--
 modules/sd_models.py |   2 +-
 modules/shared.py    |  16 ++---
 modules/ui.py        |   4 +-
 setup.py             |  40 ++++++------
 webui.py             |   8 +--
 8 files changed, 118 insertions(+), 110 deletions(-)

diff --git a/TODO.md b/TODO.md
index 0a5b0e99d..503154a59 100644
--- a/TODO.md
+++ b/TODO.md
@@ -11,13 +11,14 @@ Stuff to be fixed...
 
 Stuff to be added...
 
-- Update README
+- Update `README.md`
 - Add Gradio theme maker
 - Transformers version
 - Create new GitHub hooks/actions for CI/CD  
 - Redo Extensions tab: see <https://vladmandic.github.io/sd-extension-manager/pages/extensions.html>
 - Stream-load models as option for slow storage
 - Auto-test `torch.layer_norm` for FP16
+- Monitor file changes by misbehaving extensions
 
 ## Investigate
 
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index 9b8438800..c45e6c203 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -1,80 +1,81 @@
 import argparse
 import os
-from modules.paths_internal import data_path, sd_default_config, sd_model_file
+from modules.paths_internal import data_path
 
-parser = argparse.ArgumentParser(description="Stable Diffusion", conflict_handler='resolve', formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=55, indent_increment=2, width=200))
+parser = argparse.ArgumentParser(description="SD.Next", conflict_handler='resolve', epilog='For other options see UI Settings page', prog='', add_help=True, formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=55, indent_increment=2, width=200))
+parser._optionals = parser.add_argument_group('Other options') # pylint: disable=protected-access
+group = parser.add_argument_group('Server options')
+# group.add_argument("--config", type=str, default=sd_default_config, help=argparse.SUPPRESS)
 
-parser.add_argument("-f", action='store_true', help=argparse.SUPPRESS)  # allows running as root; implemented outside of webui
-parser.add_argument("--ui-settings-file", type=str, help=argparse.SUPPRESS, default=os.path.join(data_path, 'config.json'))
-parser.add_argument("--ui-config-file", type=str, help=argparse.SUPPRESS, default=os.path.join(data_path, 'ui-config.json'))
-parser.add_argument("--config", type=str, default=sd_default_config, help=argparse.SUPPRESS)
-parser.add_argument("--theme", type=str, help=argparse.SUPPRESS, default=None)
+group.add_argument("-f", action='store_true', help=argparse.SUPPRESS)  # allows running as root; implemented outside of webui
+group.add_argument("--ui-settings-file", type=str, help=argparse.SUPPRESS, default=os.path.join(data_path, 'config.json'))
+group.add_argument("--ui-config-file", type=str, help=argparse.SUPPRESS, default=os.path.join(data_path, 'ui-config.json'))
+group.add_argument("--hide-ui-dir-config", action='store_true', help=argparse.SUPPRESS, default=False)
+group.add_argument("--theme", type=str, help=argparse.SUPPRESS, default=None)
+group.add_argument("--disable-console-progressbars", action='store_true', help=argparse.SUPPRESS, default=True)
+group.add_argument("--disable-safe-unpickle", action='store_true', help=argparse.SUPPRESS, default=True)
+group.add_argument("--lowram", action='store_true', help=argparse.SUPPRESS)
 
-parser.add_argument("--medvram", action='store_true', help="Enable model optimizations for sacrificing a little speed for low memory usage")
-parser.add_argument("--lowvram", action='store_true', help="Enable model optimizations for sacrificing a lot of speed for lowest memory usage")
-parser.add_argument("--lowram", action='store_true', help="Load checkpoint weights to VRAM instead of RAM")
-
-parser.add_argument("--ckpt", type=str, default=sd_model_file, help="Path to checkpoint of stable diffusion model to load immediately",)
-parser.add_argument('--vae', type=str, help='Path to checkpoint of stable diffusion VAE model to load immediately', default=None)
-parser.add_argument("--data-dir", type=str, default=os.path.dirname(os.path.dirname(os.path.realpath(__file__))), help="Base path where all user data is stored")
-parser.add_argument("--models-dir", type=str, default="models", help="Nase path where all models are stored",)
-
-parser.add_argument("--allow-code", action='store_true', help="Allow custom script execution")
-parser.add_argument("--share", action='store_true', help="Enable to make the UI accessible through Gradio site")
-parser.add_argument("--enable-insecure", action='store_true', help="Enable extensions tab regardless of other options")
-parser.add_argument("--use-cpu", nargs='+', help="Force use CPU for specified modules", default=[], type=str.lower)
-parser.add_argument("--listen", action='store_true', help="Launch web server using public IP address")
-parser.add_argument("--port", type=int, help="Launch web server with given server port", default=None)
-parser.add_argument("--hide-ui-dir-config", action='store_true', help="Hide directory configuration from UI", default=False)
-parser.add_argument("--freeze-settings", action='store_true', help="Disable editing settings", default=False)
-parser.add_argument("--gradio-auth", type=str, help='Set Gradio authentication like "username:password,username:password""', default=None)
-parser.add_argument("--gradio-auth-path", type=str, help='Set Gradio authentication using file', default=None)
-parser.add_argument("--autolaunch", action='store_true', help="Open the UI URL in the system's default browser upon launch", default=False)
-parser.add_argument("--disable-console-progressbars", action='store_true', help="Do not output progressbars to console", default=True)
-parser.add_argument("--disable-safe-unpickle", action='store_true', help="Disable checking models for malicious code", default=True)
-parser.add_argument("--api-auth", type=str, help='Set API authentication', default=None)
-parser.add_argument("--api-log", action='store_true', help="Enable logging of all API requests")
-parser.add_argument("--device-id", type=str, help="Select the default CUDA device to use", default=None)
-parser.add_argument("--cors-origins", type=str, help="Allowed CORS origin(s) in the form of a comma-separated list", default=None)
-parser.add_argument("--cors-regex", type=str, help="Allowed CORS origin(s) in the form of a single regular expression", default=None)
-parser.add_argument("--tls-keyfile", type=str, help="Partially enables TLS, requires --tls-certfile to fully function", default=None)
-parser.add_argument("--tls-certfile", type=str, help="Partially enables TLS, requires --tls-keyfile to fully function", default=None)
-parser.add_argument("--server-name", type=str, help="Sets hostname of server", default=None)
-parser.add_argument("--no-hashing", action='store_true', help="Disable sha256 hashing of checkpoints", default=False)
-parser.add_argument("--no-download-sd-model", action='store_true', help="Disable download of default model even if no model is found", default=False)
-parser.add_argument("--profile", action='store_true', help="Run profiler, default: %(default)s")
-parser.add_argument("--disable-queue", action='store_true', help="Disable Gradio queues and force use of HTTP instead of WebSockets, default: %(default)s")
+group.add_argument("--config", type=str, default=os.path.join(data_path, 'config.json'), help="Use specific configuration file, default: %(default)s")
+group.add_argument("--medvram", action='store_true', help="Split model stages and keep only active part in VRAM, default: %(default)s")
+group.add_argument("--lowvram", action='store_true', help="Split model components and keep only active part in VRAM, default: %(default)s")
+group.add_argument("--ckpt", type=str, default=None, help="Path to model checkpoint to load immediately, default: %(default)s")
+group.add_argument('--vae', type=str, default=None, help='Path to VAE checkpoint to load immediately, default: %(default)s')
+group.add_argument("--data-dir", type=str, default=os.path.dirname(os.path.dirname(os.path.realpath(__file__))), help="Base path where all user data is stored, default: %(default)s")
+group.add_argument("--models-dir", type=str, default="models", help="Base path where all models are stored, default: %(default)s",)
+group.add_argument("--allow-code", action='store_true', help="Allow custom script execution, default: %(default)s")
+group.add_argument("--share", action='store_true', help="Enable UI accessible through Gradio site, default: %(default)s")
+group.add_argument("--insecure", action='store_true', help="Enable extensions tab regardless of other options, default: %(default)s")
+group.add_argument("--use-cpu", nargs='+', default=[], type=str.lower, help="Force use CPU for specified modules, default: %(default)s")
+group.add_argument("--listen", action='store_true', help="Launch web server using public IP address, default: %(default)s")
+group.add_argument("--port", type=int, default=7860, help="Launch web server with given server port, default: %(default)s")
+group.add_argument("--freeze", action='store_true', help="Disable editing settings", default=False)
+group.add_argument("--auth", type=str, help='Set access authentication like "user:pwd,user:pwd""', default=None)
+group.add_argument("--authfile", type=str, help='Set access authentication using file, default: %(default)s', default=None)
+group.add_argument("--autolaunch", action='store_true', help="Open the UI URL in the system's default browser upon launch", default=False)
+group.add_argument("--api-auth", type=str, help='Set API authentication, default: %(default)s', default=None)
+group.add_argument("--api-log", default=False, action='store_true', help="Enable logging of all API requests, default: %(default)s")
+group.add_argument("--device-id", type=str, help="Select the default CUDA device to use, default: %(default)s", default=None)
+group.add_argument("--cors-origins", type=str, help="Allowed CORS origins as comma-separated list, default: %(default)s", default=None)
+group.add_argument("--cors-regex", type=str, help="Allowed CORS origins as regular expression, default: %(default)s", default=None)
+group.add_argument("--tls-keyfile", type=str, help="Enable TLS and specify key file, default: %(default)s", default=None)
+group.add_argument("--tls-certfile", type=str, help="Enable TLS and specify cert file, default: %(default)s", default=None)
+group.add_argument("--server-name", type=str, help="Sets hostname of server, default: %(default)s", default=None)
+group.add_argument("--no-hashing", action='store_true', help="Disable hashing of checkpoints, default: %(default)s", default=False)
+group.add_argument("--no-download", action='store_true', help="Disable download of default model, default: %(default)s", default=False)
+group.add_argument("--profile", action='store_true', help="Run profiler, default: %(default)s")
+group.add_argument("--disable-queue", action='store_true', help="Disable queues, default: %(default)s")
 
 
 def compatibility_args(opts, args):
-    parser.add_argument("--ckpt-dir", type=str, help=argparse.SUPPRESS, default=opts.ckpt_dir)
-    parser.add_argument("--vae-dir", type=str, help=argparse.SUPPRESS, default=opts.vae_dir)
-    parser.add_argument("--embeddings-dir", type=str, help=argparse.SUPPRESS, default=opts.embeddings_dir)
-    parser.add_argument("--embeddings-templates-dir", type=str, help=argparse.SUPPRESS, default=opts.embeddings_templates_dir)
-    parser.add_argument("--hypernetwork-dir", type=str, help=argparse.SUPPRESS, default=opts.hypernetwork_dir)
-    parser.add_argument("--codeformer-models-path", type=str, help=argparse.SUPPRESS, default=opts.codeformer_models_path)
-    parser.add_argument("--gfpgan-models-path", type=str, help=argparse.SUPPRESS, default=opts.gfpgan_models_path)
-    parser.add_argument("--esrgan-models-path", type=str, help=argparse.SUPPRESS, default=opts.esrgan_models_path)
-    parser.add_argument("--bsrgan-models-path", type=str, help=argparse.SUPPRESS, default=opts.bsrgan_models_path)
-    parser.add_argument("--realesrgan-models-path", type=str, help=argparse.SUPPRESS, default=opts.realesrgan_models_path)
-    parser.add_argument("--scunet-models-path", help=argparse.SUPPRESS, default=opts.scunet_models_path)
-    parser.add_argument("--swinir-models-path", help=argparse.SUPPRESS, default=opts.swinir_models_path)
-    parser.add_argument("--ldsr-models-path", help=argparse.SUPPRESS, default=opts.ldsr_models_path)
-    parser.add_argument("--clip-models-path", type=str, help=argparse.SUPPRESS, default=opts.clip_models_path)
-    parser.add_argument("--disable-extension-access", default = False, action='store_true', help=argparse.SUPPRESS)
-    parser.add_argument("--opt-channelslast", help=argparse.SUPPRESS, default=opts.opt_channelslast)
-    parser.add_argument("--xformers", default = (opts.cross_attention_optimization == "xFormers"), action='store_true', help=argparse.SUPPRESS)
-    parser.add_argument("--disable-nan-check", help=argparse.SUPPRESS, default=opts.disable_nan_check)
-    parser.add_argument("--token-merging", help=argparse.SUPPRESS, default=opts.token_merging)
-    parser.add_argument("--rollback-vae", help=argparse.SUPPRESS, default=opts.rollback_vae)
-    parser.add_argument("--no-half", help=argparse.SUPPRESS, default=opts.no_half)
-    parser.add_argument("--no-half-vae", help=argparse.SUPPRESS, default=opts.no_half_vae)
-    parser.add_argument("--precision", help=argparse.SUPPRESS, default=opts.precision)
-    parser.add_argument("--api", help=argparse.SUPPRESS, default=True)
-    parser.add_argument("--sub-quad-q-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_q_chunk_size)
-    parser.add_argument("--sub-quad-kv-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_kv_chunk_size)
-    parser.add_argument("--sub-quad-chunk-threshold", help=argparse.SUPPRESS, default=opts.sub_quad_chunk_threshold)
-    parser.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
+    group.add_argument("--ckpt-dir", type=str, help=argparse.SUPPRESS, default=opts.ckpt_dir)
+    group.add_argument("--vae-dir", type=str, help=argparse.SUPPRESS, default=opts.vae_dir)
+    group.add_argument("--embeddings-dir", type=str, help=argparse.SUPPRESS, default=opts.embeddings_dir)
+    group.add_argument("--embeddings-templates-dir", type=str, help=argparse.SUPPRESS, default=opts.embeddings_templates_dir)
+    group.add_argument("--hypernetwork-dir", type=str, help=argparse.SUPPRESS, default=opts.hypernetwork_dir)
+    group.add_argument("--codeformer-models-path", type=str, help=argparse.SUPPRESS, default=opts.codeformer_models_path)
+    group.add_argument("--gfpgan-models-path", type=str, help=argparse.SUPPRESS, default=opts.gfpgan_models_path)
+    group.add_argument("--esrgan-models-path", type=str, help=argparse.SUPPRESS, default=opts.esrgan_models_path)
+    group.add_argument("--bsrgan-models-path", type=str, help=argparse.SUPPRESS, default=opts.bsrgan_models_path)
+    group.add_argument("--realesrgan-models-path", type=str, help=argparse.SUPPRESS, default=opts.realesrgan_models_path)
+    group.add_argument("--scunet-models-path", help=argparse.SUPPRESS, default=opts.scunet_models_path)
+    group.add_argument("--swinir-models-path", help=argparse.SUPPRESS, default=opts.swinir_models_path)
+    group.add_argument("--ldsr-models-path", help=argparse.SUPPRESS, default=opts.ldsr_models_path)
+    group.add_argument("--clip-models-path", type=str, help=argparse.SUPPRESS, default=opts.clip_models_path)
+    group.add_argument("--disable-extension-access", default = False, action='store_true', help=argparse.SUPPRESS)
+    group.add_argument("--opt-channelslast", help=argparse.SUPPRESS, default=opts.opt_channelslast)
+    group.add_argument("--xformers", default = (opts.cross_attention_optimization == "xFormers"), action='store_true', help=argparse.SUPPRESS)
+    group.add_argument("--disable-nan-check", help=argparse.SUPPRESS, default=opts.disable_nan_check)
+    group.add_argument("--token-merging", help=argparse.SUPPRESS, default=opts.token_merging)
+    group.add_argument("--rollback-vae", help=argparse.SUPPRESS, default=opts.rollback_vae)
+    group.add_argument("--no-half", help=argparse.SUPPRESS, default=opts.no_half)
+    group.add_argument("--no-half-vae", help=argparse.SUPPRESS, default=opts.no_half_vae)
+    group.add_argument("--precision", help=argparse.SUPPRESS, default=opts.precision)
+    group.add_argument("--api", help=argparse.SUPPRESS, default=True)
+    group.add_argument("--sub-quad-q-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_q_chunk_size)
+    group.add_argument("--sub-quad-kv-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_kv_chunk_size)
+    group.add_argument("--sub-quad-chunk-threshold", help=argparse.SUPPRESS, default=opts.sub_quad_chunk_threshold)
+    group.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
 
     opts.use_old_emphasis_implementation = False
     opts.use_old_karras_scheduler_sigmas = False
@@ -93,7 +94,7 @@ def compatibility_args(opts, args):
     opts.print_hypernet_extra = False
     opts.dimensions_and_batch_together = True
 
-    parser.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir)
+    group.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir)
     args = parser.parse_args()
     if 'lyco_dir' in args:
         args.lyco_dir = opts.lyco_dir
diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index 47640377c..459dfd091 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -92,12 +92,12 @@ def undo_optimizations():
 def fix_checkpoint():
     """checkpoints are now added and removed in embedding/hypernet code, since torch doesn't want
     checkpoints to be added when not training (there's a warning)"""
-    pass
+    pass # pylint: disable=unnecessary-pass
 
 
 def weighted_loss(sd_model, pred, target, mean=True):
     #Calculate the weight normally, but ignore the mean
-    loss = sd_model._old_get_loss(pred, target, mean=False)
+    loss = sd_model._old_get_loss(pred, target, mean=False) # pylint: disable=protected-access
 
     #Check if we have weights available
     weight = getattr(sd_model, '_custom_loss_weight', None)
@@ -110,12 +110,12 @@ def weighted_loss(sd_model, pred, target, mean=True):
 def weighted_forward(sd_model, x, c, w, *args, **kwargs):
     try:
         #Temporarily append weights to a place accessible during loss calc
-        sd_model._custom_loss_weight = w
+        sd_model._custom_loss_weight = w # pylint: disable=protected-access
 
         #Replace 'get_loss' with a weight-aware one. Otherwise we need to reimplement 'forward' completely
         #Keep 'get_loss', but don't overwrite the previous old_get_loss if it's already set
         if not hasattr(sd_model, '_old_get_loss'):
-            sd_model._old_get_loss = sd_model.get_loss
+            sd_model._old_get_loss = sd_model.get_loss # pylint: disable=protected-access
         sd_model.get_loss = MethodType(weighted_loss, sd_model)
 
         #Run the standard forward function, but with the patched 'get_loss'
@@ -129,7 +129,7 @@ def weighted_forward(sd_model, x, c, w, *args, **kwargs):
 
         #If we have an old loss function, reset the loss function to the original one
         if hasattr(sd_model, '_old_get_loss'):
-            sd_model.get_loss = sd_model._old_get_loss
+            sd_model.get_loss = sd_model._old_get_loss # pylint: disable=protected-access
             del sd_model._old_get_loss
 
 def apply_weighted_forward(sd_model):
@@ -181,6 +181,10 @@ class StableDiffusionModelHijack:
                 torch._dynamo.config.verbose = opts.cuda_compile_verbose # pylint: disable=protected-access
                 torch._dynamo.config.suppress_errors = opts.cuda_compile_errors # pylint: disable=protected-access
                 torch.backends.cudnn.benchmark = True
+                if opts.cuda_compile_mode == 'hidet':
+                    import hidet
+                    hidet.torch.dynamo_config.use_tensor_core(True)
+                    hidet.torch.dynamo_config.search_space(2)
                 m.model = torch.compile(m.model, mode="default", backend=opts.cuda_compile_mode, fullgraph=False, dynamic=False)
                 print("Model compile enabled:", opts.cuda_compile_mode)
             except Exception as err:
diff --git a/modules/sd_models.py b/modules/sd_models.py
index a2ef7a012..4f5b18891 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -118,7 +118,7 @@ def list_models():
         checkpoint_info.register()
     print(f'Available models: {shared.opts.ckpt_dir} {len(checkpoints_list)}')
     if len(checkpoints_list) == 0:
-        if not shared.cmd_opts.no_download_sd_model:
+        if not shared.cmd_opts.no_download:
             key = input('Download the default model? (y/N) ')
             if key.lower().startswith('y'):
                 model_url = "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensors"
diff --git a/modules/shared.py b/modules/shared.py
index 288804fea..aabed166c 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -49,7 +49,7 @@ ui_reorder_categories = [
     "scripts",
 ]
 
-cmd_opts.disable_extension_access = (cmd_opts.share or cmd_opts.listen or cmd_opts.server_name) and not cmd_opts.enable_insecure
+cmd_opts.disable_extension_access = (cmd_opts.share or cmd_opts.listen or cmd_opts.server_name) and not cmd_opts.insecure
 devices.device, devices.device_interrogate, devices.device_gfpgan, devices.device_esrgan, devices.device_codeformer = (devices.cpu if any(y in cmd_opts.use_cpu for y in [x, 'all']) else devices.get_optimal_device() for x in ['sd', 'interrogate', 'gfpgan', 'esrgan', 'codeformer'])
 device = devices.device
 is_device_dml = False
@@ -59,7 +59,7 @@ clip_model = None
 
 
 if device.type == 'privateuseone':
-    import modules.dml
+    import modules.dml # pylint: disable=ungrouped-imports
     is_device_dml = True
 
 
@@ -324,9 +324,9 @@ options_templates.update(options_section(('cuda', "CUDA Settings"), {
     "cuda_allow_tf32": OptionInfo(True, "Allow TF32 math ops"),
     "cuda_allow_tf16_reduced": OptionInfo(True, "Allow TF16 reduced precision math ops"),
     "cuda_compile": OptionInfo(False, "Enable model compile (experimental)"),
-    "cuda_compile_mode": OptionInfo("none", "Model compile mode (experimental)", gr.Radio, lambda: {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser']}),
-    "cuda_compile_verbose": OptionInfo(True, "Compile verbose mode"),
-    "cuda_compile_errors": OptionInfo(True, "Compile suppress errors"),
+    "cuda_compile_mode": OptionInfo("none", "Model compile mode (experimental)", gr.Radio, lambda: {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet']}),
+    "cuda_compile_verbose": OptionInfo(True, "Model compile verbose mode"),
+    "cuda_compile_errors": OptionInfo(True, "Model compile suppress errors"),
 }))
 
 options_templates.update(options_section(('upscaling', "Upscaling"), {
@@ -468,7 +468,7 @@ class Options:
     def __setattr__(self, key, value):
         if self.data is not None:
             if key in self.data or key in self.data_labels:
-                if cmd_opts.freeze_settings:
+                if cmd_opts.freeze:
                     print(f'Settings are frozen: {key}')
                     return
                 if cmd_opts.hide_ui_dir_config and key in restricted_opts:
@@ -514,7 +514,7 @@ class Options:
         return data_label.default
 
     def save(self, filename):
-        assert not cmd_opts.freeze_settings, "saving settings is disabled"
+        assert not cmd_opts.freeze, "saving settings is disabled"
         with open(filename, "w", encoding="utf8") as file:
             json.dump(self.data, file, indent=4)
 
@@ -587,7 +587,7 @@ opts = Options()
 batch_cond_uncond = opts.always_batch_cond_uncond or not (cmd_opts.lowvram or cmd_opts.medvram)
 parallel_processing_allowed = not cmd_opts.lowvram and not cmd_opts.medvram
 xformers_available = False
-config_filename = cmd_opts.ui_settings_file
+config_filename = cmd_opts.config
 os.makedirs(opts.hypernetwork_dir, exist_ok=True)
 hypernetworks = {}
 loaded_hypernetworks = []
diff --git a/modules/ui.py b/modules/ui.py
index 12e888308..e9d6c6efc 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -254,7 +254,7 @@ def setup_progressbar(*args, **kwargs): # pylint: disable=unused-argument
 def apply_setting(key, value):
     if value is None:
         return gr.update()
-    if shared.cmd_opts.freeze_settings:
+    if shared.cmd_opts.freeze:
         return gr.update()
     # dont allow model to be swapped when model hash exists in prompt
     if key == "sd_model_checkpoint" and opts.disable_weights_auto_swap:
@@ -1292,7 +1292,7 @@ def create_ui():
                     current_row = gr.Column(variant='compact')
                     current_row.__enter__()
                     previous_section = item.section
-                if k in quicksettings_names and not shared.cmd_opts.freeze_settings:
+                if k in quicksettings_names and not shared.cmd_opts.freeze:
                     quicksettings_list.append((i, k, item))
                     components.append(dummy_component)
                 elif section_must_be_skipped:
diff --git a/setup.py b/setup.py
index cfaa1aaa0..3d680f397 100644
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,7 @@ class Dot(dict): # dot notation access to dictionary attributes
 
 
 log = logging.getLogger("sd")
-args = Dot({ 'debug': False, 'upgrade': False, 'noupdate': False, 'nodirectml': False, 'skip-extensions': False, 'skip-requirements': False, 'reset': False })
+args = Dot({ 'debug': False, 'upgrade': False, 'skip_update': False, 'no_directml': False, 'skip_extensions': False, 'skip_requirements': False, 'reset': False })
 quick_allowed = True
 errors = 0
 opts = {}
@@ -169,7 +169,6 @@ def clone(url, folder, commithash=None):
 
 # check python version
 def check_python():
-    import platform
     supported_minors = [9, 10]
     if args.experimental:
         supported_minors.append(11)
@@ -199,7 +198,7 @@ def check_torch():
         xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
     else:
         machine = platform.machine()
-        if 'arm' not in machine and 'aarch' not in machine and not args.nodirectml: # torch-directml is available on AMD64
+        if 'arm' not in machine and 'aarch' not in machine and not args.no_directml: # torch-directml is available on AMD64
             log.info('Using DirectML Backend')
             torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.0 torchvision torch-directml')
             xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
@@ -223,7 +222,7 @@ def check_torch():
                 log.info(f'Torch detected GPU: {torch.cuda.get_device_name(device)} VRAM {round(torch.cuda.get_device_properties(device).total_memory / 1024 / 1024)} Arch {torch.cuda.get_device_capability(device)} Cores {torch.cuda.get_device_properties(device).multi_processor_count}')
         else:
             try:
-                import torch_directml
+                import torch_directml # pylint: disable=import-error
                 import pkg_resources
                 version = pkg_resources.get_distribution("torch-directml")
                 log.info(f'Torch backend: DirectML ({version})')
@@ -245,6 +244,8 @@ def check_torch():
         install(tensorflow_package, 'tensorflow', ignore=True)
     except Exception as e:
         log.debug(f'Cannot install tensorflow package: {e}')
+    if opts.get('cuda_compile_mode', '') == 'hidet':
+        install('hidet', 'hidet')
 
 
 # install required packages
@@ -322,7 +323,7 @@ def install_extensions():
         extensions = list_extensions(folder)
         log.info(f'Extensions enabled: {extensions}')
         for ext in extensions:
-            if not args.noupdate:
+            if not args.skip_update:
                 try:
                     update(os.path.join(folder, ext))
                 except:
@@ -346,7 +347,7 @@ def install_submodules():
         git('checkout master')
         log.info('Continuing setup')
     txt = git('submodule --quiet update --init --recursive')
-    if not args.noupdate:
+    if not args.skip_update:
         log.info('Updating submodules')
         submodules = git('submodule').splitlines()
         for submodule in submodules:
@@ -461,7 +462,7 @@ def check_version():
 
 
 def update_wiki():
-    if not args.noupdate:
+    if not args.skip_update:
         log.info('Updating Wiki')
         try:
             update(os.path.join(os.path.dirname(__file__), "wiki"))
@@ -503,16 +504,17 @@ def check_timestamp():
 
 
 def add_args():
-    parser.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
-    parser.add_argument('--reset', default = False, action='store_true', help = "Reset main repository to latest version, default: %(default)s")
-    parser.add_argument('--upgrade', default = False, action='store_true', help = "Upgrade main repository to latest version, default: %(default)s")
-    parser.add_argument('--noupdate', default = False, action='store_true', help = "Skip update of extensions and submodules, default: %(default)s")
-    parser.add_argument('--nodirectml', default = False, action='store_true', help = "Although nVidia and AMD toolkit aren't detected, use CPU not DirectML, default: %(default)s")
-    parser.add_argument('--skip-requirements', default = False, action='store_true', help = "Skips checking and installing requirements, default: %(default)s")
-    parser.add_argument('--skip-extensions', default = False, action='store_true', help = "Skips running individual extension installers, default: %(default)s")
-    parser.add_argument('--skip-git', default = False, action='store_true', help = "Skips running all GIT operations, default: %(default)s")
-    parser.add_argument('--experimental', default = False, action='store_true', help = "Allow unsupported versions of libraries, default: %(default)s")
-    parser.add_argument('--test', default = False, action='store_true', help = "Run test only, default: %(default)s")
+    group = parser.add_argument_group('Setup options')
+    group.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
+    group.add_argument('--reset', default = False, action='store_true', help = "Reset main repository to latest version, default: %(default)s")
+    group.add_argument('--upgrade', default = False, action='store_true', help = "Upgrade main repository to latest version, default: %(default)s")
+    group.add_argument('--no-directml', default = False, action='store_true', help = "Use CPU instead of DirectML if no compatible GPU is detected, default: %(default)s")
+    group.add_argument('--skip-update', default = False, action='store_true', help = "Skip update of extensions and submodules, default: %(default)s")
+    group.add_argument('--skip-requirements', default = False, action='store_true', help = "Skips checking and installing requirements, default: %(default)s")
+    group.add_argument('--skip-extensions', default = False, action='store_true', help = "Skips running individual extension installers, default: %(default)s")
+    group.add_argument('--skip-git', default = False, action='store_true', help = "Skips running all GIT operations, default: %(default)s")
+    group.add_argument('--experimental', default = False, action='store_true', help = "Allow unsupported versions of libraries, default: %(default)s")
+    group.add_argument('--test', default = False, action='store_true', help = "Run test only, default: %(default)s")
 
 
 def parse_args():
@@ -550,8 +552,8 @@ def git_reset():
 
 def read_options():
     global opts # pylint: disable=global-statement
-    if os.path.isfile(args.ui_settings_file):
-        with open(args.ui_settings_file, "r", encoding="utf8") as file:
+    if os.path.isfile(args.config):
+        with open(args.config, "r", encoding="utf8") as file:
             opts = json.load(file)
 
 
diff --git a/webui.py b/webui.py
index 7ca052a2e..6d78be03c 100644
--- a/webui.py
+++ b/webui.py
@@ -198,10 +198,10 @@ def start_ui():
         shared.demo.queue(16)
 
     gradio_auth_creds = []
-    if cmd_opts.gradio_auth:
-        gradio_auth_creds += [x.strip() for x in cmd_opts.gradio_auth.strip('"').replace('\n', '').split(',') if x.strip()]
-    if cmd_opts.gradio_auth_path:
-        with open(cmd_opts.gradio_auth_path, 'r', encoding="utf8") as file:
+    if cmd_opts.auth:
+        gradio_auth_creds += [x.strip() for x in cmd_opts.auth.strip('"').replace('\n', '').split(',') if x.strip()]
+    if cmd_opts.authfile:
+        with open(cmd_opts.authfile, 'r', encoding="utf8") as file:
             for line in file.readlines():
                 gradio_auth_creds += [x.strip() for x in line.split(',') if x.strip()]
 

From de8d0bef9f64cab4e83845e2055d671181f5b4a6 Mon Sep 17 00:00:00 2001
From: Disty0 <semihgulec2005@gmail.com>
Date: Sun, 30 Apr 2023 18:19:37 +0300
Subject: [PATCH 56/69] More patches and Import IPEX after Torch

---
 extensions-builtin/sd-webui-controlnet        |  2 +-
 modules/api/api.py                            | 19 ++++++++++++++++++-
 modules/codeformer/codeformer_arch.py         |  4 ++++
 modules/codeformer/vqgan_arch.py              |  4 ++++
 modules/codeformer_model.py                   |  7 +++++--
 modules/deepbooru.py                          |  4 ++++
 modules/deepbooru_model.py                    |  4 ++++
 modules/devices.py                            |  3 +--
 modules/esrgan_model.py                       |  4 ++++
 modules/esrgan_model_arch.py                  |  4 ++++
 modules/extras.py                             |  4 ++++
 modules/hypernetworks/hypernetwork.py         | 17 ++++++++++++++---
 modules/interrogate.py                        |  4 ++++
 modules/lowvram.py                            |  4 ++++
 modules/mac_specific.py                       |  4 ++++
 modules/memmon.py                             | 13 +++++++------
 modules/models/diffusion/ddpm_edit.py         |  4 ++++
 modules/models/diffusion/uni_pc/sampler.py    |  4 ++++
 modules/models/diffusion/uni_pc/uni_pc.py     |  4 ++++
 modules/processing.py                         | 10 ++++++----
 modules/prompt_parser.py                      |  4 ++++
 modules/safe.py                               |  4 ++++
 modules/sd_disable_initialization.py          |  4 ++++
 modules/sd_hijack.py                          |  4 ++++
 modules/sd_hijack_clip.py                     |  4 ++++
 modules/sd_hijack_inpainting.py               |  4 ++++
 modules/sd_hijack_open_clip.py                |  4 ++++
 modules/sd_hijack_optimizations.py            |  8 ++++++--
 modules/sd_hijack_unet.py                     |  4 ++++
 modules/sd_hijack_xlmr.py                     |  4 ++++
 modules/sd_models.py                          |  4 ++++
 modules/sd_models_config.py                   |  4 ++++
 modules/sd_samplers_common.py                 |  4 ++++
 modules/sd_samplers_compvis.py                |  4 ++++
 modules/sd_samplers_kdiffusion.py             |  4 ++++
 modules/sd_vae.py                             |  8 +++++++-
 modules/sd_vae_approx.py                      |  4 ++++
 modules/sub_quadratic_attention.py            |  4 ++++
 modules/textual_inversion/dataset.py          |  4 ++++
 modules/textual_inversion/image_embedding.py  |  4 ++++
 .../textual_inversion/textual_inversion.py    |  5 ++++-
 modules/xlmr.py                               |  4 ++++
 webui.py                                      |  4 ++++
 wiki                                          |  2 +-
 44 files changed, 202 insertions(+), 24 deletions(-)

diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index 09d1fcbf4..d2da774a4 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit 09d1fcbf4dc715bca7547496f850801f95f732a6
+Subproject commit d2da774a40ff9c3770e21f71fb516403022fc3f6
diff --git a/modules/api/api.py b/modules/api/api.py
index 0717edfaf..fdd26f868 100644
--- a/modules/api/api.py
+++ b/modules/api/api.py
@@ -573,7 +573,24 @@ class Api:
             ram = { 'error': f'{err}' }
         try:
             import torch
-            if torch.cuda.is_available():
+            if shared.cmd_opts.use_ipex():
+                import intel_extension_for_pytorch as ipex
+                system = { 'free': (torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), 'used': torch.xpu.memory_allocated(), 'total': torch.xpu.get_device_properties("xpu").total_memory }
+                s = dict(torch.xpu.memory_stats("xpu"))
+                allocated = { 'current': s['allocated_bytes.all.current'], 'peak': s['allocated_bytes.all.peak'] }
+                reserved = { 'current': s['reserved_bytes.all.current'], 'peak': s['reserved_bytes.all.peak'] }
+                active = { 'current': s['active_bytes.all.current'], 'peak': s['active_bytes.all.peak'] }
+                inactive = { 'current': s['inactive_split_bytes.all.current'], 'peak': s['inactive_split_bytes.all.peak'] }
+                warnings = { 'retries': s['num_alloc_retries'], 'oom': s['num_ooms'] }
+                cuda = {
+                    'system': system,
+                    'active': active,
+                    'allocated': allocated,
+                    'reserved': reserved,
+                    'inactive': inactive,
+                    'events': warnings,
+                }
+            elif torch.cuda.is_available():
                 s = torch.cuda.mem_get_info()
                 system = { 'free': s[0], 'used': s[1] - s[0], 'total': s[1] }
                 s = dict(torch.cuda.memory_stats(shared.device))
diff --git a/modules/codeformer/codeformer_arch.py b/modules/codeformer/codeformer_arch.py
index 11dcc3ee7..6d7b926fe 100644
--- a/modules/codeformer/codeformer_arch.py
+++ b/modules/codeformer/codeformer_arch.py
@@ -3,6 +3,10 @@
 import math
 import numpy as np
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from torch import nn, Tensor
 import torch.nn.functional as F
 from typing import Optional, List
diff --git a/modules/codeformer/vqgan_arch.py b/modules/codeformer/vqgan_arch.py
index e72936838..e66bb2a72 100644
--- a/modules/codeformer/vqgan_arch.py
+++ b/modules/codeformer/vqgan_arch.py
@@ -7,6 +7,10 @@ https://github.com/samb-t/unleashing-transformers/blob/master/models/vqgan.py
 '''
 import numpy as np
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.nn as nn
 import torch.nn.functional as F
 import copy
diff --git a/modules/codeformer_model.py b/modules/codeformer_model.py
index 5217f69db..9d75e823d 100644
--- a/modules/codeformer_model.py
+++ b/modules/codeformer_model.py
@@ -3,6 +3,10 @@ import sys
 
 import cv2
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 import modules.face_restoration
 from modules import shared, devices, modelloader, errors
@@ -103,8 +107,7 @@ def setup_model(dirname):
                             output = self.net(cropped_face_t, w=w if w is not None else shared.opts.code_former_weight, adain=True)[0]
                             restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1))
                         del output
-                        from modules import shared
-                        if shared.cmd_opts.use_ipex:
+                        if cmd_opts.use_ipex:
                             torch.xpu.empty_cache()
                         else:
                             torch.cuda.empty_cache()
diff --git a/modules/deepbooru.py b/modules/deepbooru.py
index 1c4554a20..50e400fd8 100644
--- a/modules/deepbooru.py
+++ b/modules/deepbooru.py
@@ -2,6 +2,10 @@ import os
 import re
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import numpy as np
 
 from modules import modelloader, paths, deepbooru_model, devices, images, shared
diff --git a/modules/deepbooru_model.py b/modules/deepbooru_model.py
index c2c77cd25..ef53494a2 100644
--- a/modules/deepbooru_model.py
+++ b/modules/deepbooru_model.py
@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.nn as nn
 import torch.nn.functional as F
 
diff --git a/modules/devices.py b/modules/devices.py
index 1ab082f33..8be1e3866 100644
--- a/modules/devices.py
+++ b/modules/devices.py
@@ -5,8 +5,7 @@ from modules import shared
 try:
     import intel_extension_for_pytorch as ipex
 except:
-    if shared.cmd_opts.use_ipex:
-        print("Failed to import IPEX")
+    pass
 
 if sys.platform == "darwin":
     from modules import mac_specific
diff --git a/modules/esrgan_model.py b/modules/esrgan_model.py
index bb4c6619b..769d66f01 100644
--- a/modules/esrgan_model.py
+++ b/modules/esrgan_model.py
@@ -2,6 +2,10 @@ import os
 
 import numpy as np
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from PIL import Image
 from basicsr.utils.download_util import load_file_from_url
 
diff --git a/modules/esrgan_model_arch.py b/modules/esrgan_model_arch.py
index 411d98d38..fc352d0ba 100644
--- a/modules/esrgan_model_arch.py
+++ b/modules/esrgan_model_arch.py
@@ -2,6 +2,10 @@
 
 import math
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.nn as nn
 import torch.nn.functional as F
 
diff --git a/modules/extras.py b/modules/extras.py
index c0ae9477f..4513f2491 100644
--- a/modules/extras.py
+++ b/modules/extras.py
@@ -4,6 +4,10 @@ import html
 import shutil
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import tqdm
 import gradio as gr
 import safetensors.torch
diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py
index 4aa5ffcdc..a1caecbe4 100644
--- a/modules/hypernetworks/hypernetwork.py
+++ b/modules/hypernetworks/hypernetwork.py
@@ -8,6 +8,10 @@ import inspect
 
 import modules.textual_inversion.dataset
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import tqdm
 from einops import rearrange, repeat
 from ldm.util import default
@@ -591,7 +595,10 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
             print("Cannot resume from saved optimizer!")
             print(e)
 
-    scaler = torch.cuda.amp.GradScaler()
+    if shared.cmd_opts.use_ipex:
+        scaler = torch.xpu.amp.GradScaler()
+    else:
+        scaler = torch.cuda.amp.GradScaler()
     
     batch_size = ds.batch_size
     gradient_step = ds.gradient_step
@@ -708,7 +715,9 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
                     hypernetwork.eval()
                     rng_state = torch.get_rng_state()
                     cuda_rng_state = None
-                    if torch.cuda.is_available():
+                    if shared.cmd_opts.use_ipex:
+                        cuda_rng_state = torch.xpu.get_rng_state_all()
+                    elif torch.cuda.is_available():
                         cuda_rng_state = torch.cuda.get_rng_state_all()
                     shared.sd_model.cond_stage_model.to(devices.device)
                     shared.sd_model.first_stage_model.to(devices.device)
@@ -745,7 +754,9 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
                         shared.sd_model.cond_stage_model.to(devices.cpu)
                         shared.sd_model.first_stage_model.to(devices.cpu)
                     torch.set_rng_state(rng_state)
-                    if torch.cuda.is_available():
+                    if shared.cmd_opts.use_ipex:
+                        torch.xpu.set_rng_state_all(cuda_rng_state)
+                    elif torch.cuda.is_available():
                         torch.cuda.set_rng_state_all(cuda_rng_state)
                     hypernetwork.train()
                     if image is not None:
diff --git a/modules/interrogate.py b/modules/interrogate.py
index 6afbde570..93bb08f20 100644
--- a/modules/interrogate.py
+++ b/modules/interrogate.py
@@ -5,6 +5,10 @@ from pathlib import Path
 import re
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.hub
 
 from torchvision import transforms
diff --git a/modules/lowvram.py b/modules/lowvram.py
index e254cc131..7dba01593 100644
--- a/modules/lowvram.py
+++ b/modules/lowvram.py
@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from modules import devices
 
 module_in_gpu = None
diff --git a/modules/mac_specific.py b/modules/mac_specific.py
index c8a534d0e..2455800d5 100644
--- a/modules/mac_specific.py
+++ b/modules/mac_specific.py
@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import platform
 from modules.sd_hijack_utils import CondFunc
 from packaging import version
diff --git a/modules/memmon.py b/modules/memmon.py
index 4ceb29a37..3abc70ac3 100644
--- a/modules/memmon.py
+++ b/modules/memmon.py
@@ -2,6 +2,12 @@ import threading
 import time
 from collections import defaultdict
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
+
+from modules import shared
 
 
 class MemUsageMonitor(threading.Thread):
@@ -19,7 +25,6 @@ class MemUsageMonitor(threading.Thread):
         self.daemon = True
         self.run_flag = threading.Event()
         self.data = defaultdict(int)
-        from modules import shared
         if not torch.cuda.is_available() or not shared.cmd_opts.use_ipex:
             self.disabled = True
         else:
@@ -40,9 +45,8 @@ class MemUsageMonitor(threading.Thread):
                     self.disabled = True
 
     def cuda_mem_get_info(self):
-        from modules import shared
         if shared.cmd_opts.use_ipex:
-            return torch.xpu.mem_get_info("xpu")
+            return [(torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), torch.xpu.get_device_properties("xpu").total_memory]
         else:
             index = self.device.index if self.device.index is not None else torch.cuda.current_device()
             return torch.cuda.mem_get_info(index)
@@ -52,7 +56,6 @@ class MemUsageMonitor(threading.Thread):
             return
         while True:
             self.run_flag.wait()
-            from modules import shared
             if shared.cmd_opts.use_ipex:
                 torch.xpu.reset_peak_memory_stats()
             else:
@@ -72,7 +75,6 @@ class MemUsageMonitor(threading.Thread):
         for k, v in self.read().items():
             print(k, -(v // -(1024 ** 2)))
         print(self, 'raw torch memory stats:')
-        from modules import shared
         if shared.cmd_opts.use_ipex:
             tm = torch.xpu.memory_stats("xpu")
         else:
@@ -95,7 +97,6 @@ class MemUsageMonitor(threading.Thread):
             self.data["free"] = free
             self.data["total"] = total
 
-            from modules import shared
             if shared.cmd_opts.use_ipex:
                 torch_stats = torch.xpu.memory_stats("xpu")
             else:
diff --git a/modules/models/diffusion/ddpm_edit.py b/modules/models/diffusion/ddpm_edit.py
index f3d49c44c..846a74fc4 100644
--- a/modules/models/diffusion/ddpm_edit.py
+++ b/modules/models/diffusion/ddpm_edit.py
@@ -10,6 +10,10 @@ https://github.com/CompVis/taming-transformers
 # See more details in LICENSE.
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.nn as nn
 import numpy as np
 import pytorch_lightning as pl
diff --git a/modules/models/diffusion/uni_pc/sampler.py b/modules/models/diffusion/uni_pc/sampler.py
index 3100522ab..6dd7c7fd8 100644
--- a/modules/models/diffusion/uni_pc/sampler.py
+++ b/modules/models/diffusion/uni_pc/sampler.py
@@ -2,6 +2,10 @@
 
 import numpy as np
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 from .uni_pc import NoiseScheduleVP, model_wrapper, UniPC
 from modules import shared, devices
diff --git a/modules/models/diffusion/uni_pc/uni_pc.py b/modules/models/diffusion/uni_pc/uni_pc.py
index 61ee39522..895fc58c3 100644
--- a/modules/models/diffusion/uni_pc/uni_pc.py
+++ b/modules/models/diffusion/uni_pc/uni_pc.py
@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.nn.functional as F
 import math
 import time
diff --git a/modules/processing.py b/modules/processing.py
index 293a8d606..36737fdbe 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -8,6 +8,10 @@ from typing import Any, Dict, List
 
 import psutil
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import numpy as np
 from PIL import Image, ImageFilter, ImageOps
 import cv2
@@ -55,10 +59,8 @@ def memory_stats():
     except Exception as e:
         mem.update({ 'ram': e })
     try:
-        from modules import shared
-        if shared.cmd_opts.use_ipex:
-            s = torch.xpu.mem_get_info()
-            gpu = { 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }
+        if cmd_opts.use_ipex:
+            gpu = { 'used': gb(torch.xpu.memory_allocated()), 'total': gb(torch.xpu.get_device_properties("xpu").total_memory) }
             s = dict(torch.xpu.memory_stats("xpu"))
             mem.update({
                 'gpu': gpu,
diff --git a/modules/prompt_parser.py b/modules/prompt_parser.py
index 7006f2822..6722d9f80 100644
--- a/modules/prompt_parser.py
+++ b/modules/prompt_parser.py
@@ -368,3 +368,7 @@ if __name__ == "__main__":
     doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
 else:
     import torch  # doctest faster
+    try:
+        import intel_extension_for_pytorch as ipex
+    except:
+        pass
diff --git a/modules/safe.py b/modules/safe.py
index 9a1133ddc..dd463ccdd 100644
--- a/modules/safe.py
+++ b/modules/safe.py
@@ -6,6 +6,10 @@ import zipfile
 import re
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import numpy
 import _codecs
 
diff --git a/modules/sd_disable_initialization.py b/modules/sd_disable_initialization.py
index c4a09d15d..5cc5e4e7a 100644
--- a/modules/sd_disable_initialization.py
+++ b/modules/sd_disable_initialization.py
@@ -1,6 +1,10 @@
 import ldm.modules.encoders.modules
 import open_clip
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import transformers.utils.hub
 
 
diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index f817b7afd..6d2bb3b0f 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -1,6 +1,10 @@
 from types import MethodType
 from rich import print # pylint: disable=redefined-builtin
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from torch.nn.functional import silu
 import ldm.modules.attention
 import ldm.modules.diffusionmodules.model
diff --git a/modules/sd_hijack_clip.py b/modules/sd_hijack_clip.py
index 945f7732d..cf4abf84f 100644
--- a/modules/sd_hijack_clip.py
+++ b/modules/sd_hijack_clip.py
@@ -2,6 +2,10 @@ import math
 from collections import namedtuple
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 from modules import prompt_parser, devices, sd_hijack
 from modules.shared import opts
diff --git a/modules/sd_hijack_inpainting.py b/modules/sd_hijack_inpainting.py
index 4b23c132d..1a9ea9b4c 100644
--- a/modules/sd_hijack_inpainting.py
+++ b/modules/sd_hijack_inpainting.py
@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 import ldm.models.diffusion.ddpm
 import ldm.models.diffusion.ddim
diff --git a/modules/sd_hijack_open_clip.py b/modules/sd_hijack_open_clip.py
index f76fc1f3b..c0c204a82 100644
--- a/modules/sd_hijack_open_clip.py
+++ b/modules/sd_hijack_open_clip.py
@@ -1,5 +1,9 @@
 import open_clip.tokenizer
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 from modules import sd_hijack_clip, devices
 
diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py
index 5168b4b7a..3887e238d 100644
--- a/modules/sd_hijack_optimizations.py
+++ b/modules/sd_hijack_optimizations.py
@@ -2,6 +2,10 @@ import math
 import psutil
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from torch import einsum
 
 from ldm.util import default
@@ -26,7 +30,7 @@ def get_available_vram():
         stats = torch.xpu.memory_stats("xpu")
         mem_active = stats['active_bytes.all.current']
         mem_reserved = stats['reserved_bytes.all.current']
-        mem_free_xpu, _ = torch.xpu.mem_get_info("xpu")
+        mem_free_xpu, _ = [(torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), torch.xpu.get_device_properties("xpu").total_memory]
         mem_free_torch = mem_reserved - mem_active
         mem_free_total = mem_free_xpu + mem_free_torch
         return mem_free_total
@@ -201,7 +205,7 @@ def einsum_op_cuda(q, k, v):
         stats = torch.xpu.memory_stats("xpu")
         mem_active = stats['active_bytes.all.current']
         mem_reserved = stats['reserved_bytes.all.current']
-        mem_free_xpu, _ = torch.xpu.mem_get_info("xpu")
+        mem_free_xpu, _ = [(torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), torch.xpu.get_device_properties("xpu").total_memory]
         mem_free_torch = mem_reserved - mem_active
         mem_free_total = mem_free_xpu + mem_free_torch
         # Divide factor of safety as there's copying and fragmentation
diff --git a/modules/sd_hijack_unet.py b/modules/sd_hijack_unet.py
index 7ff553ae3..ce6ac1306 100644
--- a/modules/sd_hijack_unet.py
+++ b/modules/sd_hijack_unet.py
@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from packaging import version
 
 from modules import devices
diff --git a/modules/sd_hijack_xlmr.py b/modules/sd_hijack_xlmr.py
index 28528329b..a9cb9454c 100644
--- a/modules/sd_hijack_xlmr.py
+++ b/modules/sd_hijack_xlmr.py
@@ -1,4 +1,8 @@
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 from modules import sd_hijack_clip, devices
 
diff --git a/modules/sd_models.py b/modules/sd_models.py
index 49de37097..2e2b5f0a2 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -8,6 +8,10 @@ from os import mkdir
 from urllib import request
 from rich import print, progress # pylint: disable=redefined-builtin
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import safetensors.torch
 from omegaconf import OmegaConf
 import tomesd
diff --git a/modules/sd_models_config.py b/modules/sd_models_config.py
index a9c515b14..5bc3799a0 100644
--- a/modules/sd_models_config.py
+++ b/modules/sd_models_config.py
@@ -1,6 +1,10 @@
 import os
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 from modules import paths, sd_disable_initialization
 
diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py
index 888f9a30e..dfb478251 100644
--- a/modules/sd_samplers_common.py
+++ b/modules/sd_samplers_common.py
@@ -1,6 +1,10 @@
 from collections import namedtuple
 import numpy as np
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from PIL import Image
 from modules import devices, processing, images, sd_vae_approx
 
diff --git a/modules/sd_samplers_compvis.py b/modules/sd_samplers_compvis.py
index 8de719323..6f08a9022 100644
--- a/modules/sd_samplers_compvis.py
+++ b/modules/sd_samplers_compvis.py
@@ -4,6 +4,10 @@ import ldm.models.diffusion.plms
 
 import numpy as np
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 
 from modules.shared import state
 from modules import sd_samplers_common, prompt_parser, shared
diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py
index a30d351fc..5ba34cc33 100644
--- a/modules/sd_samplers_kdiffusion.py
+++ b/modules/sd_samplers_kdiffusion.py
@@ -1,6 +1,10 @@
 from collections import deque
 import inspect
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import k_diffusion.sampling
 from modules import prompt_parser, devices, sd_samplers_common
 
diff --git a/modules/sd_vae.py b/modules/sd_vae.py
index e5c544487..a13d73be7 100644
--- a/modules/sd_vae.py
+++ b/modules/sd_vae.py
@@ -3,8 +3,14 @@ import collections
 import glob
 from copy import deepcopy
 from rich import print # pylint: disable=redefined-builtin
+from modules import shared
 import torch
-from modules import paths, shared, devices, script_callbacks, sd_models
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    if shared.cmd_opts.use_ipex:
+        print("Failed to import IPEX")
+from modules import paths, devices, script_callbacks, sd_models
 
 
 vae_ignore_keys = {"model_ema.decay", "model_ema.num_updates"}
diff --git a/modules/sd_vae_approx.py b/modules/sd_vae_approx.py
index e2f004683..56c3fb15f 100644
--- a/modules/sd_vae_approx.py
+++ b/modules/sd_vae_approx.py
@@ -1,6 +1,10 @@
 import os
 
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from torch import nn
 from modules import devices, paths
 
diff --git a/modules/sub_quadratic_attention.py b/modules/sub_quadratic_attention.py
index 87c18a38d..0af680de2 100644
--- a/modules/sub_quadratic_attention.py
+++ b/modules/sub_quadratic_attention.py
@@ -14,6 +14,10 @@ from functools import partial
 import math
 from typing import Optional, NamedTuple, List
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from torch import Tensor
 from torch.utils.checkpoint import checkpoint
 
diff --git a/modules/textual_inversion/dataset.py b/modules/textual_inversion/dataset.py
index af9fbcf28..272ae76ea 100644
--- a/modules/textual_inversion/dataset.py
+++ b/modules/textual_inversion/dataset.py
@@ -2,6 +2,10 @@ import os
 import numpy as np
 import PIL
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from PIL import Image
 from torch.utils.data import Dataset, DataLoader, Sampler
 from torchvision import transforms
diff --git a/modules/textual_inversion/image_embedding.py b/modules/textual_inversion/image_embedding.py
index 0ba5db8a4..a2c518af3 100644
--- a/modules/textual_inversion/image_embedding.py
+++ b/modules/textual_inversion/image_embedding.py
@@ -4,6 +4,10 @@ import numpy as np
 import zlib
 from PIL import Image, PngImagePlugin, ImageDraw, ImageFont
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 from modules.shared import opts
 
 
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index 36a1e1e17..377b577f0 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -3,6 +3,10 @@ import html
 import csv
 from collections import namedtuple
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import tqdm
 import safetensors.torch
 from rich import print # pylint: disable=redefined-builtin
@@ -434,7 +438,6 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
         else:
             print("No saved optimizer exists in checkpoint")
 
-    from modules import shared
     if shared.cmd_opts.use_ipex:
         scaler = torch.xpu.amp.GradScaler()
     else:
diff --git a/modules/xlmr.py b/modules/xlmr.py
index 9da3161cc..a891beb6d 100644
--- a/modules/xlmr.py
+++ b/modules/xlmr.py
@@ -1,5 +1,9 @@
 from typing import Optional
 import torch
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torch.nn as nn
 from transformers import XLMRobertaModel,XLMRobertaTokenizer, BertPreTrainedModel, BertModel, BertConfig # pylint: disable=unused-import
 from transformers.models.xlm_roberta.configuration_xlm_roberta import XLMRobertaConfig
diff --git a/webui.py b/webui.py
index 7ca052a2e..e9ec96c6d 100644
--- a/webui.py
+++ b/webui.py
@@ -12,6 +12,10 @@ from modules import timer, errors
 startup_timer = timer.Timer()
 
 import torch # pylint: disable=C0411
+try:
+    import intel_extension_for_pytorch as ipex
+except:
+    pass
 import torchvision # pylint: disable=W0611,C0411
 import pytorch_lightning # pytorch_lightning should be imported after torch, but it re-enables warnings on import so import once to disable them # pylint: disable=W0611,C0411
 logging.getLogger("xformers").addFilter(lambda record: 'A matching Triton is not available' not in record.getMessage())
diff --git a/wiki b/wiki
index 6cd8fde16..4cbdffaa9 160000
--- a/wiki
+++ b/wiki
@@ -1 +1 @@
-Subproject commit 6cd8fde165190057c0849fa6f8dbb183f717b176
+Subproject commit 4cbdffaa95978d0a46758eac4a3fbe689eb4cdcd

From 56cdac65929578fd5900ecdad7a8c450e24832bd Mon Sep 17 00:00:00 2001
From: Disty0 <semihgulec2005@gmail.com>
Date: Sun, 30 Apr 2023 18:36:52 +0300
Subject: [PATCH 57/69] undo cli

---
 cli/modules/bench.py                   |  7 ++--
 cli/modules/interrogate-offline.py     | 18 ++--------
 cli/modules/lora-extract.py            | 16 ++-------
 cli/modules/lora-latents.py            | 11 +-----
 cli/modules/util.py                    | 20 +----------
 cli/random/dynamotest.py               | 49 +++++++-------------------
 cli/train-lora.py                      | 13 +------
 cli/train/latents.py                   | 11 +-----
 extensions-builtin/sd-webui-controlnet |  2 +-
 9 files changed, 24 insertions(+), 123 deletions(-)

diff --git a/cli/modules/bench.py b/cli/modules/bench.py
index 18791bfc9..094b73f63 100755
--- a/cli/modules/bench.py
+++ b/cli/modules/bench.py
@@ -10,7 +10,7 @@ import time
 from PIL import Image
 import sdapi
 from util import Map, log
-from modules import shared
+
 
 options = Map({
     'restore_faces': False,
@@ -56,10 +56,7 @@ async def txt2img():
 def memstats():
     mem = sdapi.getsync('/sdapi/v1/memory')
     cpu = mem.get('ram', 'unavailable')
-    if shared.cmd_opts.use_ipex:
-        gpu = mem.get('xpu', 'unavailable')
-    else:
-        gpu = mem.get('cuda', 'unavailable')
+    gpu = mem.get('cuda', 'unavailable')
     if 'active' in gpu:
         gpu['session'] = gpu.pop('active')
     if 'reserved' in gpu:
diff --git a/cli/modules/interrogate-offline.py b/cli/modules/interrogate-offline.py
index c2623cda6..6d9ae56fa 100755
--- a/cli/modules/interrogate-offline.py
+++ b/cli/modules/interrogate-offline.py
@@ -6,12 +6,6 @@ import json
 import time
 import argparse
 import torch
-from modules import shared
-try:
-    import intel_extension_for_pytorch as ipex
-except:
-    if shared.cmd_opts.use_ipex:
-        print("Failed to import IPEX")
 import filetype
 from PIL import Image
 import transformers
@@ -25,10 +19,7 @@ model = None
 processor = None
 extractor = None
 dtype = torch.float32
-if shared.cmd_opts.use_ipex:
-    device = torch.device('xpu')
-else:
-    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
 options = Map({
     'input': '',
@@ -138,12 +129,7 @@ def unload_model():
         del extractor
         extractor = None
     gc.collect()
-    if shared.cmd_opts.use_ipex:
-        with torch.no_grad():
-            torch.xpu.empty_cache()
-        with torch.xpu.device('xpu'):
-            torch.xpu.empty_cache()
-    elif torch.cuda.is_available():
+    if torch.cuda.is_available():
         with torch.no_grad():
             torch.cuda.empty_cache()
         with torch.cuda.device('cuda'):
diff --git a/cli/modules/lora-extract.py b/cli/modules/lora-extract.py
index 9a781789a..102728308 100755
--- a/cli/modules/lora-extract.py
+++ b/cli/modules/lora-extract.py
@@ -10,12 +10,6 @@ import sys
 import time
 import argparse
 import torch
-from modules import shared
-try:
-    import intel_extension_for_pytorch as ipex
-except:
-    if shared.cmd_opts.use_ipex:
-        print("Failed to import IPEX")
 import transformers
 from tqdm import tqdm
 from util import log
@@ -26,10 +20,7 @@ import networks.lora as lora
 
 
 def svd(args): # pylint: disable=redefined-outer-name
-    if shared.cmd_opts.use_ipex:
-        device = torch.device('xpu')
-    else:
-        device = 'cuda' if torch.cuda.is_available() and args.device == 'cuda' else 'cpu'
+    device = 'cuda' if torch.cuda.is_available() and args.device == 'cuda' else 'cpu'
     transformers.logging.set_verbosity_error()
     CLAMP_QUANTILE = 0.99
     MIN_DIFF = 1e-6
@@ -47,10 +38,7 @@ def svd(args): # pylint: disable=redefined-outer-name
     log.info({ 'loading model': args.tuned })
     text_encoder_t, _, unet_t = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.tuned)
     with torch.no_grad():
-        if shared.cmd_opts.use_ipex:
-            torch.xpu.empty_cache()
-        else:
-            torch.cuda.empty_cache()
+        torch.cuda.empty_cache()
     # create LoRA network to extract weights: Use dim (rank) as alpha
     lora_network_o = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_o, unet_o)
     lora_network_t = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_t, unet_t)
diff --git a/cli/modules/lora-latents.py b/cli/modules/lora-latents.py
index 7e701df12..d556d596b 100755
--- a/cli/modules/lora-latents.py
+++ b/cli/modules/lora-latents.py
@@ -10,12 +10,6 @@ import warnings
 import cv2
 import numpy as np
 import torch
-from modules import shared
-try:
-    import intel_extension_for_pytorch as ipex
-except:
-    if shared.cmd_opts.use_ipex:
-        print("Failed to import IPEX")
 from PIL import Image
 from torchvision import transforms
 from tqdm import tqdm
@@ -26,10 +20,7 @@ import library.model_util as model_util
 import library.train_util as train_util
 
 warnings.filterwarnings('ignore')
-if shared.cmd_opts.use_ipex:
-    device = torch.device('xpu')
-else:
-    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 options = Map({
   'batch': 1,
   'input': '',
diff --git a/cli/modules/util.py b/cli/modules/util.py
index d48887961..5ab9dee6b 100755
--- a/cli/modules/util.py
+++ b/cli/modules/util.py
@@ -45,25 +45,7 @@ def get_memory():
     try:
         import torch
         from modules import shared
-        if shared.cmd_opts.use_ipex:
-            import intel_extension_for_pytorch as ipex
-            s = torch.xpu.mem_get_info()
-            gpu = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }
-            s = dict(torch.xpu.memory_stats('xpu'))
-            allocated = { 'current': gb(s['allocated_bytes.all.current']), 'peak': gb(s['allocated_bytes.all.peak']) }
-            reserved = { 'current': gb(s['reserved_bytes.all.current']), 'peak': gb(s['reserved_bytes.all.peak']) }
-            active = { 'current': gb(s['active_bytes.all.current']), 'peak': gb(s['active_bytes.all.peak']) }
-            inactive = { 'current': gb(s['inactive_split_bytes.all.current']), 'peak': gb(s['inactive_split_bytes.all.peak']) }
-            warnings = { 'retries': s['num_alloc_retries'], 'oom': s['num_ooms'] }
-            mem.update({
-                'gpu': gpu,
-                'gpu-active': active,
-                'gpu-allocated': allocated,
-                'gpu-reserved': reserved,
-                'gpu-inactive': inactive,
-                'events': warnings,
-            })
-        elif torch.cuda.is_available():
+        if torch.cuda.is_available():
             s = torch.cuda.mem_get_info()
             gpu = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }
             s = dict(torch.cuda.memory_stats('cuda'))
diff --git a/cli/random/dynamotest.py b/cli/random/dynamotest.py
index 556ae96a8..82b1143c6 100755
--- a/cli/random/dynamotest.py
+++ b/cli/random/dynamotest.py
@@ -7,14 +7,9 @@ import warnings
 
 import numpy as np
 import torch
-from modules import shared
-try:
-    import intel_extension_for_pytorch as ipex
-except:
-    if shared.cmd_opts.use_ipex:
-        print("Failed to import IPEX")
 from torchvision.models import resnet18
 
+
 print('torch:', torch.__version__)
 try:
     import torch._dynamo as dynamo # must be imported explicitly or namespace is not found
@@ -29,42 +24,24 @@ warnings.filterwarnings('ignore', category=UserWarning) # disable those for now
 
 
 def timed(fn): # returns the result of running `fn()` and the time it took for `fn()` to run in ms using CUDA events
-    if shared.cmd_opts.use_ipex:
-        start = torch.xpu.Event(enable_timing=True)
-        end = torch.xpu.Event(enable_timing=True)
-        start.record()
-        result = fn()
-        end.record()
-        torch.xpu.synchronize()
-        return result, start.elapsed_time(end)
-    else:
-        start = torch.cuda.Event(enable_timing=True)
-        end = torch.cuda.Event(enable_timing=True)
-        start.record()
-        result = fn()
-        end.record()
-        torch.cuda.synchronize()
-        return result, start.elapsed_time(end)
+    start = torch.cuda.Event(enable_timing=True)
+    end = torch.cuda.Event(enable_timing=True)
+    start.record()
+    result = fn()
+    end.record()
+    torch.cuda.synchronize()
+    return result, start.elapsed_time(end)
 
 
 def generate_data(b):
-    if shared.cmd_opts.use_ipex:
-        return (
-            torch.randn(b, 3, 128, 128).to(torch.float32).xpu(),
-            torch.randint(1000, (b,)).xpu(),
-        )
-    else:
-        return (
-            torch.randn(b, 3, 128, 128).to(torch.float32).cuda(),
-            torch.randint(1000, (b,)).cuda(),
-        )
+    return (
+        torch.randn(b, 3, 128, 128).to(torch.float32).cuda(),
+        torch.randint(1000, (b,)).cuda(),
+    )
 
 
 def init_model():
-    if shared.cmd_opts.use_ipex:
-        return resnet18().to(torch.float32).xpu()
-    else:
-        return resnet18().to(torch.float32).cuda()
+    return resnet18().to(torch.float32).cuda()
 
 
 def eval(mod, inp):
diff --git a/cli/train-lora.py b/cli/train-lora.py
index f1e295a35..6f82063a6 100755
--- a/cli/train-lora.py
+++ b/cli/train-lora.py
@@ -23,12 +23,6 @@ import shutil
 import argparse
 import tempfile
 import torch
-from modules import shared
-try:
-    import intel_extension_for_pytorch as ipex
-except:
-    if shared.cmd_opts.use_ipex:
-        print("Failed to import IPEX")
 import logging
 import importlib
 import transformers
@@ -123,12 +117,7 @@ options = Map({
 
 def mem_stats():
     gc.collect()
-    if shared.cmd_opts.use_ipex:
-        with torch.no_grad():
-            torch.xpu.empty_cache()
-        with torch.xpu.device('xpu'):
-            torch.cuda.empty_cache()
-    elif torch.cuda.is_available():
+    if torch.cuda.is_available():
         with torch.no_grad():
             torch.cuda.empty_cache()
         with torch.cuda.device('cuda'):
diff --git a/cli/train/latents.py b/cli/train/latents.py
index 715f92aba..94249b18a 100755
--- a/cli/train/latents.py
+++ b/cli/train/latents.py
@@ -10,12 +10,6 @@ import warnings
 import cv2
 import numpy as np
 import torch
-from modules import shared
-try:
-    import intel_extension_for_pytorch as ipex
-except:
-    if shared.cmd_opts.use_ipex:
-        print("Failed to import IPEX")
 from PIL import Image
 from torchvision import transforms
 from tqdm import tqdm
@@ -34,10 +28,7 @@ import library.model_util as model_util
 import library.train_util as train_util
 
 warnings.filterwarnings('ignore')
-if shared.cmd_opts.use_ipex:
-    device = torch.device('xpu')
-else:
-    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 options = Map({
   'batch': 1,
   'input': '',
diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index d2da774a4..af4720780 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit d2da774a40ff9c3770e21f71fb516403022fc3f6
+Subproject commit af4720780f10d912789cbd6db1fbc6d2f0afc533

From 185b796991e6ae8f71c03fea826581f0799ce9b4 Mon Sep 17 00:00:00 2001
From: Disty0 <semihgulec2005@gmail.com>
Date: Sun, 30 Apr 2023 18:37:58 +0300
Subject: [PATCH 58/69] undo cli

---
 cli/modules/util.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cli/modules/util.py b/cli/modules/util.py
index 5ab9dee6b..479b77233 100755
--- a/cli/modules/util.py
+++ b/cli/modules/util.py
@@ -44,7 +44,6 @@ def get_memory():
         mem.update({ 'ram': e })
     try:
         import torch
-        from modules import shared
         if torch.cuda.is_available():
             s = torch.cuda.mem_get_info()
             gpu = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }

From e157681985207b6c987ebc6ca9bde701495e658d Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sun, 30 Apr 2023 11:43:56 -0400
Subject: [PATCH 59/69] update todo

---
 TODO.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/TODO.md b/TODO.md
index 503154a59..040fca0a2 100644
--- a/TODO.md
+++ b/TODO.md
@@ -15,10 +15,11 @@ Stuff to be added...
 - Add Gradio theme maker
 - Transformers version
 - Create new GitHub hooks/actions for CI/CD  
-- Redo Extensions tab: see <https://vladmandic.github.io/sd-extension-manager/pages/extensions.html>
+- Redo Extensions tab: <https://vladmandic.github.io/sd-extension-manager/pages/extensions.html>
 - Stream-load models as option for slow storage
 - Auto-test `torch.layer_norm` for FP16
 - Monitor file changes by misbehaving extensions
+- Kitchen theme: <https://github.com/canisminor1990/sd-webui-kitchen-theme>
 
 ## Investigate
 

From d62ee69c75fe79c9452d51b0997cf5e91efb7dca Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sun, 30 Apr 2023 12:01:25 -0400
Subject: [PATCH 60/69] cleanup installer

---
 setup.py => installer.py |  8 ++++----
 launch.py                | 40 +++++++++++++++++++++-------------------
 modules/cmd_args.py      |  3 +--
 modules/shared.py        |  4 ++--
 4 files changed, 28 insertions(+), 27 deletions(-)
 rename setup.py => installer.py (98%)

diff --git a/setup.py b/installer.py
similarity index 98%
rename from setup.py
rename to installer.py
index 393663088..3be52a6c2 100644
--- a/setup.py
+++ b/installer.py
@@ -13,7 +13,6 @@ except:
     import argparse
     parser = argparse.ArgumentParser(description="Stable Diffusion", conflict_handler='resolve', formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=55, indent_increment=2, width=200))
 
-
 class Dot(dict): # dot notation access to dictionary attributes
     __getattr__ = dict.get
     __setattr__ = dict.__setitem__
@@ -224,9 +223,9 @@ def check_torch():
         import torch
         log.info(f'Torch {torch.__version__}')
         if shared.cmd_opts.use_ipex:
-                import intel_extension_for_pytorch as ipex
-                log.info(f'Torch backend: Intel OneAPI {torch.__version__}')
-                log.info(f'Torch detected GPU: {torch.xpu.get_device_name("xpu")} VRAM {round(torch.xpu.get_device_properties("xpu").total_memory / 1024 / 1024)}')
+            import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
+            log.info(f'Torch backend: Intel OneAPI {torch.__version__}')
+            log.info(f'Torch detected GPU: {torch.xpu.get_device_name("xpu")} VRAM {round(torch.xpu.get_device_properties("xpu").total_memory / 1024 / 1024)}')
         elif torch.cuda.is_available():
             if torch.version.cuda:
                 log.info(f'Torch backend: nVidia CUDA {torch.version.cuda} cuDNN {torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else "N/A"}')
@@ -524,6 +523,7 @@ def add_args():
     group.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
     group.add_argument('--reset', default = False, action='store_true', help = "Reset main repository to latest version, default: %(default)s")
     group.add_argument('--upgrade', default = False, action='store_true', help = "Upgrade main repository to latest version, default: %(default)s")
+    group.add_argument("--use-ipex", action='store_true', help="Use Intel OneAPI XPU backend, default: %(default)s", default=False)
     group.add_argument('--no-directml', default = False, action='store_true', help = "Use CPU instead of DirectML if no compatible GPU is detected, default: %(default)s")
     group.add_argument('--skip-update', default = False, action='store_true', help = "Skip update of extensions and submodules, default: %(default)s")
     group.add_argument('--skip-requirements', default = False, action='store_true', help = "Skips checking and installing requirements, default: %(default)s")
diff --git a/launch.py b/launch.py
index 8df3f787c..32e2f8419 100644
--- a/launch.py
+++ b/launch.py
@@ -1,23 +1,24 @@
+### majority of this file is superflous, but used by some extensions as helpers during extension installation
+
 import subprocess
 import os
 import sys
 import shlex
 import logging
-import setup
-import modules.paths_internal
-import modules.cmd_args
-
-setup.ensure_base_requirements()
-from rich import print # pylint: disable=redefined-builtin,wrong-import-order
-
-### majority of this file is superflous, but used by some extensions as helpers during extension installation
 
 commandline_args = os.environ.get('COMMANDLINE_ARGS', "")
 sys.argv += shlex.split(commandline_args)
-setup.add_args()
-setup.extensions_preload(force=False)
-setup.parse_args()
+
+import installer
+installer.add_args()
+installer.ensure_base_requirements()
+installer.extensions_preload(force=False)
+installer.parse_args()
+
+import modules.cmd_args
 args, _ = modules.cmd_args.parser.parse_known_args()
+
+import modules.paths_internal
 script_path = modules.paths_internal.script_path
 extensions_dir = modules.paths_internal.extensions_dir
 git = os.environ.get('GIT', "git")
@@ -41,6 +42,7 @@ def commit_hash():
 
 def run(command, desc=None, errdesc=None, custom_env=None, live=False):
     if desc is not None:
+        from rich import print # pylint: disable=redefined-builtin,wrong-import-order
         print(desc)
     if live:
         result = subprocess.run(command, check=False, shell=True, env=os.environ if custom_env is None else custom_env)
@@ -62,7 +64,7 @@ def check_run(command):
 
 
 def is_installed(package):
-    return setup.installed(package)
+    return installer.installed(package)
 
 
 def repo_dir(name):
@@ -85,20 +87,20 @@ def check_run_python(code):
 
 
 def git_clone(url, tgt, _name, commithash=None):
-    setup.clone(url, tgt, commithash)
+    installer.clone(url, tgt, commithash)
 
 
 def run_extension_installer(ext_dir):
-    setup.run_extension_installer(ext_dir)
+    installer.run_extension_installer(ext_dir)
 
 if __name__ == "__main__":
-    setup.run_setup()
-    setup.extensions_preload(force=True)
-    setup.log.info(f"Server arguments: {sys.argv[1:]}")
-    setup.log.debug('Starting WebUI')
+    installer.run_setup()
+    installer.extensions_preload(force=True)
+    installer.log.info(f"Server arguments: {sys.argv[1:]}")
+    installer.log.debug('Starting WebUI')
     logging.disable(logging.INFO)
     if args.test:
-        setup.log.info("Test only")
+        installer.log.info("Test only")
         import webui
         exit(0)
     import webui
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index 0d1da94ef..d365c067a 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -27,7 +27,6 @@ group.add_argument("--allow-code", action='store_true', help="Allow custom scrip
 group.add_argument("--share", action='store_true', help="Enable UI accessible through Gradio site, default: %(default)s")
 group.add_argument("--insecure", action='store_true', help="Enable extensions tab regardless of other options, default: %(default)s")
 group.add_argument("--use-cpu", nargs='+', default=[], type=str.lower, help="Force use CPU for specified modules, default: %(default)s")
-group.add_argument("--use-ipex", action='store_true', help="Force use Intel OneAPI XPU backend, default: %(default)s", default=False)
 group.add_argument("--listen", action='store_true', help="Launch web server using public IP address, default: %(default)s")
 group.add_argument("--port", type=int, default=7860, help="Launch web server with given server port, default: %(default)s")
 group.add_argument("--freeze", action='store_true', help="Disable editing settings", default=False)
@@ -97,6 +96,6 @@ def compatibility_args(opts, args):
 
     group.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir)
     args = parser.parse_args()
-    if 'lyco_dir' in args:
+    if 'lyco_dir' in args: # pylint disable=unsupported-membership-test
         args.lyco_dir = opts.lyco_dir
     return args
diff --git a/modules/shared.py b/modules/shared.py
index 11177d0ac..b2a93542c 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -12,11 +12,11 @@ import modules.devices as devices
 from modules import errors, ui_components, shared_items, cmd_args
 from modules.paths_internal import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611
 import modules.paths_internal as paths
-from setup import log as setup_log # pylint: disable=E0611
+from installer import log as central_logger # pylint: disable=E0611
 
 errors.install(gr)
 demo: gr.Blocks = None
-log = setup_log
+log = central_logger
 parser = cmd_args.parser
 url = 'https://github.com/vladmandic/automatic'
 if os.environ.get('IGNORE_CMD_ARGS_ERRORS', None) is None:

From 7eb82e26273cc850c647c4c37903e043b231c064 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sun, 30 Apr 2023 12:21:32 -0400
Subject: [PATCH 61/69] remove circular imports from installer

---
 installer.py        | 24 ++++++++----------------
 launch.py           |  1 -
 modules/cmd_args.py |  3 +--
 webui.py            |  2 +-
 4 files changed, 10 insertions(+), 20 deletions(-)

diff --git a/installer.py b/installer.py
index 3be52a6c2..8be8980e1 100644
--- a/installer.py
+++ b/installer.py
@@ -55,7 +55,6 @@ def setup_logging(clean=False):
 # check if package is installed
 def installed(package, friendly: str = None):
     import pkg_resources
-    from modules import shared
     ok = True
     try:
         if friendly:
@@ -76,7 +75,7 @@ def installed(package, friendly: str = None):
             ok = ok and spec is not None
             if ok:
                 version = pkg_resources.get_distribution(p[0]).version
-                if shared.cmd_opts.use_ipex and p[0] == "pytorch_lightning":
+                if args.use_ipex and p[0] == "pytorch_lightning":
                     p[1] = "1.8.6"
                 log.debug(f"Package version found: {p[0]} {version}")
                 if len(p) > 1:
@@ -93,8 +92,7 @@ def installed(package, friendly: str = None):
 
 # install package using pip if not already installed
 def install(package, friendly: str = None, ignore: bool = False):
-    from modules import shared
-    if shared.cmd_opts.use_ipex and package == "pytorch_lightning==1.9.4":
+    if args.use_ipex and package == "pytorch_lightning==1.9.4":
         package = "pytorch_lightning==1.8.6"
     def pip(arg: str):
         arg = arg.replace('>=', '==')
@@ -192,7 +190,6 @@ def check_python():
 
 # check torch version
 def check_torch():
-    from modules import shared
     if shutil.which('nvidia-smi') is not None or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe')):
         log.info('nVidia toolkit detected')
         torch_command = os.environ.get('TORCH_COMMAND', 'torch torchaudio torchvision --index-url https://download.pytorch.org/whl/cu118')
@@ -202,8 +199,7 @@ def check_torch():
         os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', '10.3.0')
         torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.4.2')
         xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
-    elif shutil.which('sycl-ls') is not None or os.path.exists('/opt/intel/oneapi'):
-        shared.cmd_opts.use_ipex = True
+    elif shutil.which('sycl-ls') is not None or os.path.exists('/opt/intel/oneapi') or args.use_ipex:
         log.info('Intel toolkit detected')
         torch_command = os.environ.get('TORCH_COMMAND', 'torch==1.13.0a0+git6c9b55e torchvision==0.14.1a0 intel_extension_for_pytorch==1.13.120+xpu --index-url https://developer.intel.com/ipex-whl-stable-xpu')
         xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
@@ -222,7 +218,7 @@ def check_torch():
     try:
         import torch
         log.info(f'Torch {torch.__version__}')
-        if shared.cmd_opts.use_ipex:
+        if args.use_ipex:
             import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
             log.info(f'Torch backend: Intel OneAPI {torch.__version__}')
             log.info(f'Torch detected GPU: {torch.xpu.get_device_name("xpu")} VRAM {round(torch.xpu.get_device_properties("xpu").total_memory / 1024 / 1024)}')
@@ -373,15 +369,11 @@ def install_submodules():
                 log.error(f'Error updating submodule: {submodule}')
 
 
-def ensure_package(pkg):
-    try:
-        import pkg # type: ignore
-    except ImportError:
-        install(pkg)
-
-
 def ensure_base_requirements():
-    ensure_package('rich')
+    try:
+        import rich # pylint: disable=unused-import
+    except ImportError:
+        install('rich', 'rich')
 
 
 def install_requirements():
diff --git a/launch.py b/launch.py
index 32e2f8419..76745b4c2 100644
--- a/launch.py
+++ b/launch.py
@@ -17,7 +17,6 @@ installer.parse_args()
 
 import modules.cmd_args
 args, _ = modules.cmd_args.parser.parse_known_args()
-
 import modules.paths_internal
 script_path = modules.paths_internal.script_path
 extensions_dir = modules.paths_internal.extensions_dir
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index d365c067a..4f74e5ad0 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -95,7 +95,6 @@ def compatibility_args(opts, args):
     opts.dimensions_and_batch_together = True
 
     group.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir)
+    group.add_argument("--lyco-dir", help=argparse.SUPPRESS, default=opts.lyco_dir)
     args = parser.parse_args()
-    if 'lyco_dir' in args: # pylint disable=unsupported-membership-test
-        args.lyco_dir = opts.lyco_dir
     return args
diff --git a/webui.py b/webui.py
index 707585ca7..8d9e0dc2f 100644
--- a/webui.py
+++ b/webui.py
@@ -13,7 +13,7 @@ startup_timer = timer.Timer()
 
 import torch # pylint: disable=C0411
 try:
-    import intel_extension_for_pytorch as ipex
+    import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
 except:
     pass
 import torchvision # pylint: disable=W0611,C0411

From 04f4da00134aef4f6ba477d9f333f71ef45be07a Mon Sep 17 00:00:00 2001
From: cool-bigdogs-tshirt
 <131823432+cool-bigdogs-tshirt@users.noreply.github.com>
Date: Sun, 30 Apr 2023 12:27:52 -0500
Subject: [PATCH 62/69] fix unipc img2img denoising sample count

was wrongly using the inverse of the intended value. smaller denoising
strength should run fewer steps.
---
 modules/models/diffusion/uni_pc/sampler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/models/diffusion/uni_pc/sampler.py b/modules/models/diffusion/uni_pc/sampler.py
index 6dd7c7fd8..953e786db 100644
--- a/modules/models/diffusion/uni_pc/sampler.py
+++ b/modules/models/diffusion/uni_pc/sampler.py
@@ -47,7 +47,7 @@ class UniPCSampler(object):
 
         # actual number of steps we'll run
         self.steps = max(
-                num_inference_steps - init_timestep,
+                init_timestep,
                 shared.opts.uni_pc_order+1,
             )
 

From a136a8ea63f2246cac5a08e2e835c158ff1897c6 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sun, 30 Apr 2023 18:44:12 -0400
Subject: [PATCH 63/69] update

---
 installer.py        | 10 ++++------
 scripts/xyz_grid.py |  3 ---
 style.css           | 33 +++++++++++++++++----------------
 3 files changed, 21 insertions(+), 25 deletions(-)

diff --git a/installer.py b/installer.py
index 8be8980e1..a7760f81e 100644
--- a/installer.py
+++ b/installer.py
@@ -75,8 +75,6 @@ def installed(package, friendly: str = None):
             ok = ok and spec is not None
             if ok:
                 version = pkg_resources.get_distribution(p[0]).version
-                if args.use_ipex and p[0] == "pytorch_lightning":
-                    p[1] = "1.8.6"
                 log.debug(f"Package version found: {p[0]} {version}")
                 if len(p) > 1:
                     ok = ok and version == p[1]
@@ -191,17 +189,17 @@ def check_python():
 # check torch version
 def check_torch():
     if shutil.which('nvidia-smi') is not None or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe')):
-        log.info('nVidia toolkit detected')
+        log.info('nVidia CUDA toolkit detected')
         torch_command = os.environ.get('TORCH_COMMAND', 'torch torchaudio torchvision --index-url https://download.pytorch.org/whl/cu118')
         xformers_package = os.environ.get('XFORMERS_PACKAGE', 'xformers==0.0.17' if opts.get('cross_attention_optimization', '') == 'xFormers' else 'none')
     elif shutil.which('rocminfo') is not None or os.path.exists('/opt/rocm/bin/rocminfo'):
-        log.info('AMD toolkit detected')
+        log.info('AMD ROCm toolkit detected')
         os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', '10.3.0')
         torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.4.2')
         xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
     elif shutil.which('sycl-ls') is not None or os.path.exists('/opt/intel/oneapi') or args.use_ipex:
-        log.info('Intel toolkit detected')
-        torch_command = os.environ.get('TORCH_COMMAND', 'torch==1.13.0a0+git6c9b55e torchvision==0.14.1a0 intel_extension_for_pytorch==1.13.120+xpu --index-url https://developer.intel.com/ipex-whl-stable-xpu')
+        log.info('Intel OneAPI Toolkit detected')
+        torch_command = os.environ.get('TORCH_COMMAND', 'torch==1.13.0a0+git6c9b55e torchvision==0.14.1a0 intel_extension_for_pytorch==1.13.120+xpu -f https://developer.intel.com/ipex-whl-stable-xpu')
         xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
     else:
         machine = platform.machine()
diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py
index d216d2fef..389560501 100644
--- a/scripts/xyz_grid.py
+++ b/scripts/xyz_grid.py
@@ -446,9 +446,6 @@ class Script(scripts.Script):
             current_values = axis_values_dropdown
             if has_choices:
                 choices = choices()
-                if len(choices) > 12:
-                    has_choices = False
-            if has_choices:
                 if isinstance(current_values,str):
                     current_values = current_values.split(",")
                 current_values = list(filter(lambda x: x in choices, current_values))
diff --git a/style.css b/style.css
index 996ad15d2..c0fe31146 100644
--- a/style.css
+++ b/style.css
@@ -1,28 +1,29 @@
 :root, .dark{ --checkbox-label-gap: 0.25em 0.1em; --section-header-text-size: 12pt; --block-background-fill: transparent;}
-.block.padded:not(.gradio-accordion) { padding: 0 !important; }
 div.gradio-container{ max-width: unset !important; }
-.hidden{ display: none; }
-.compact{ background: transparent !important; padding: 0 !important; }
 div.form{ border-width: 0; box-shadow: none; background: transparent; overflow: visible; gap: 0.5em; }
-.block.gradio-dropdown, .block.gradio-slider, .block.gradio-checkbox, .block.gradio-textbox, .block.gradio-radio, .block.gradio-checkboxgroup, .block.gradio-number, .block.gradio-colorpicker { border-width: 0 !important; box-shadow: none !important;}
-.gap.compact{ padding: 0; gap: 0.2em 0; }
 div.compact{ gap: 1em; }
-.gradio-dropdown label span:not(.has-info), .gradio-textbox label span:not(.has-info), .gradio-number label span:not(.has-info) { margin-bottom: 0; }
-.gradio-dropdown ul.options{ z-index: 3000; min-width: fit-content; max-width: inherit; white-space: nowrap; }
-.gradio-dropdown ul.options li.item { padding: 0.05em 0; }
-.gradio-dropdown ul.options li.item:not(:has(.hide)) { background-color: var(--neutral-100); }
+div.gradio-html.min{ min-height: 0; }
+.block.gradio-checkbox { margin: 0.75em 1.5em 0 0; }
+.block.gradio-dropdown, .block.gradio-slider, .block.gradio-checkbox, .block.gradio-textbox, .block.gradio-radio, .block.gradio-checkboxgroup, .block.gradio-number, .block.gradio-colorpicker { border-width: 0 !important; box-shadow: none !important;}
+.block.gradio-gallery{ background: var(--input-background-fill); }
+.block.padded:not(.gradio-accordion) { padding: 0 !important; }
+.compact{ background: transparent !important; padding: 0 !important; }
 .dark .gradio-dropdown ul.options li.item:not(:has(.hide)) { background-color: var(--neutral-900); }
-.gradio-dropdown div.wrap.wrap.wrap.wrap{ box-shadow: 0 1px 2px 0 rgba(0, 0, 0, 0.05); }
-.gradio-dropdown:not(.multiselect) .wrap-inner.wrap-inner.wrap-inner{ flex-wrap: unset; }
+.gap.compact{ padding: 0; gap: 0.2em 0; }
+.gradio-container .prose a, .gradio-container .prose a:visited{ color: unset; text-decoration: none; }
 .gradio-dropdown .single-select{ white-space: nowrap; overflow: hidden; }
 .gradio-dropdown .token-remove.remove-all.remove-all{ display: none; }
+.gradio-dropdown div.wrap.wrap.wrap.wrap{ box-shadow: 0 1px 2px 0 rgba(0, 0, 0, 0.05); }
+.gradio-dropdown label span:not(.has-info), .gradio-textbox label span:not(.has-info), .gradio-number label span:not(.has-info) { margin-bottom: 0; }
+.gradio-dropdown ul.options li.item { padding: 0.05em 0; }
+.gradio-dropdown ul.options li.item:not(:has(.hide)) { background-color: var(--neutral-100); }
+.gradio-dropdown ul.options{ z-index: 3000; min-width: fit-content; max-width: inherit; white-space: nowrap; }
+.gradio-dropdown:not(.multiselect) .wrap-inner.wrap-inner.wrap-inner{ flex-wrap: unset; }
 .gradio-dropdown.multiselect .token-remove.remove-all.remove-all{ display: flex; }
-.gradio-slider input[type="number"]{ width: 6em; }
-.block.gradio-checkbox { margin: 0.75em 1.5em 0 0; }
+.gradio-dropdown.multiselect div.wrap-inner {	overflow-x: hidden; overflow-y: auto; max-height: 50vh; overflow-wrap: anywhere; }
 .gradio-html div.wrap{ height: 100%; }
-div.gradio-html.min{ min-height: 0; }
-.block.gradio-gallery{ background: var(--input-background-fill); }
-.gradio-container .prose a, .gradio-container .prose a:visited{ color: unset; text-decoration: none; }
+.gradio-slider input[type="number"]{ width: 6em; }
+.hidden{ display: none; }
 
 /* general styled components */
 .gradio-button.tool{ max-width: 2.2em;  min-width: 2.2em !important; height: 2.4em; align-self: end; line-height: 1em; border-radius: 0.5em; }

From dedd3ffafbdf519d77399e314330cd9fbe2343a0 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sun, 30 Apr 2023 20:08:44 -0400
Subject: [PATCH 64/69] fix script api

---
 .../stable-diffusion-webui-images-browser     |  2 +-
 modules/api/api.py                            | 27 ++++--
 modules/processing.py                         | 89 ++-----------------
 3 files changed, 25 insertions(+), 93 deletions(-)

diff --git a/extensions-builtin/stable-diffusion-webui-images-browser b/extensions-builtin/stable-diffusion-webui-images-browser
index 2c988c08c..84cb61749 160000
--- a/extensions-builtin/stable-diffusion-webui-images-browser
+++ b/extensions-builtin/stable-diffusion-webui-images-browser
@@ -1 +1 @@
-Subproject commit 2c988c08c7fc2f1c0f572bc4209f0baa1fac4fee
+Subproject commit 84cb6174983812da2dff242fb484431b4ae3b8f8
diff --git a/modules/api/api.py b/modules/api/api.py
index fdd26f868..2912e2cf2 100644
--- a/modules/api/api.py
+++ b/modules/api/api.py
@@ -181,19 +181,28 @@ class Api:
                     script_args[script.args_from:script.args_to] = ui_default_values
         return script_args
 
-    def init_script_args(self, p, request, default_script_args, script_runner):
+    def init_script_args(self, p, request, default_script_args, selectable_scripts, selectable_script_idx, script_runner):
         script_args = default_script_args.copy()
+        # position 0 in script_arg is the idx+1 of the selectable script that is going to be run when using scripts.scripts_*2img.run()
+        if selectable_scripts:
+            # TODO this can corrupt values for other scripts
+            script_args[selectable_scripts.args_from:selectable_scripts.args_to] = request.script_args
+            script_args[0] = selectable_script_idx + 1
+        # Now check for always on scripts
         if request.alwayson_scripts and (len(request.alwayson_scripts) > 0):
             for alwayson_script_name in request.alwayson_scripts.keys():
                 alwayson_script = self.get_script(alwayson_script_name, script_runner)
                 if alwayson_script is None:
-                    raise HTTPException(status_code=422, detail=f"always on script {alwayson_script_name} not found")
+                    raise HTTPException(status_code=422, detail=f"Always on script not found: {alwayson_script_name}")
                 if not alwayson_script.alwayson:
-                    raise HTTPException(status_code=422, detail="Cannot have a selectable script in the always on scripts params")
+                    raise HTTPException(status_code=422, detail=f"Selectable script cannot be in always on params: {alwayson_script_name}")
                 if "args" in request.alwayson_scripts[alwayson_script_name]:
-                    p.per_script_args[alwayson_script.title()] = request.alwayson_scripts[alwayson_script_name]["args"] + script_args
+                    # TODO this can corrupt values for other scripts
+                    script_args[alwayson_script.args_from:alwayson_script.args_to] = request.alwayson_scripts[alwayson_script_name]["args"]
+                    p.per_script_args[alwayson_script.title()] = request.alwayson_scripts[alwayson_script_name]["args"]
         return script_args
 
+
     def text2imgapi(self, txt2imgreq: StableDiffusionTxt2ImgProcessingAPI):
         script_runner = scripts.scripts_txt2img
         if not script_runner.scripts:
@@ -201,7 +210,7 @@ class Api:
             ui.create_ui()
         if not self.default_script_arg_txt2img:
             self.default_script_arg_txt2img = self.init_default_script_args(script_runner)
-        selectable_scripts, _selectable_script_idx = self.get_selectable_script(txt2imgreq.script_name, script_runner)
+        selectable_scripts, selectable_script_idx = self.get_selectable_script(txt2imgreq.script_name, script_runner)
         populate = txt2imgreq.copy(update={  # Override __init__ params
             "sampler_name": validate_sampler_name(txt2imgreq.sampler_name or txt2imgreq.sampler_index),
             "do_not_save_samples": not txt2imgreq.save_images,
@@ -222,7 +231,7 @@ class Api:
             p.outpath_grids = opts.outdir_grids or opts.outdir_txt2img_grids
             p.outpath_samples = opts.outdir_samples or opts.outdir_txt2img_samples
             shared.state.begin()
-            script_args = self.init_script_args(p, txt2imgreq, self.default_script_arg_txt2img, script_runner)
+            script_args = self.init_script_args(p, txt2imgreq, self.default_script_arg_txt2img, selectable_scripts, selectable_script_idx, script_runner)
             if selectable_scripts is not None:
                 processed = scripts.scripts_txt2img.run(p, *script_args) # Need to pass args as list here
             else:
@@ -246,7 +255,7 @@ class Api:
             ui.create_ui()
         if not self.default_script_arg_img2img:
             self.default_script_arg_img2img = self.init_default_script_args(script_runner)
-        selectable_scripts, _selectable_script_idx = self.get_selectable_script(img2imgreq.script_name, script_runner)
+        selectable_scripts, selectable_script_idx = self.get_selectable_script(img2imgreq.script_name, script_runner)
         populate = img2imgreq.copy(update={  # Override __init__ params
             "sampler_name": validate_sampler_name(img2imgreq.sampler_name or img2imgreq.sampler_index),
             "do_not_save_samples": not img2imgreq.save_images,
@@ -270,7 +279,7 @@ class Api:
             p.outpath_grids = opts.outdir_img2img_grids
             p.outpath_samples = opts.outdir_img2img_samples
             shared.state.begin()
-            script_args = self.init_script_args(p, img2imgreq, self.default_script_arg_txt2img, script_runner)
+            script_args = self.init_script_args(p, img2imgreq, self.default_script_arg_img2img, selectable_scripts, selectable_script_idx, script_runner)
             if selectable_scripts is not None:
                 processed = scripts.scripts_img2img.run(p, *script_args) # Need to pass args as list here
             else:
@@ -574,7 +583,7 @@ class Api:
         try:
             import torch
             if shared.cmd_opts.use_ipex():
-                import intel_extension_for_pytorch as ipex
+                import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
                 system = { 'free': (torch.xpu.get_device_properties("xpu").total_memory - torch.xpu.memory_allocated()), 'used': torch.xpu.memory_allocated(), 'total': torch.xpu.get_device_properties("xpu").total_memory }
                 s = dict(torch.xpu.memory_stats("xpu"))
                 allocated = { 'current': s['allocated_bytes.all.current'], 'peak': s['allocated_bytes.all.peak'] }
diff --git a/modules/processing.py b/modules/processing.py
index 36737fdbe..41be44010 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -9,7 +9,7 @@ from typing import Any, Dict, List
 import psutil
 import torch
 try:
-    import intel_extension_for_pytorch as ipex
+    import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
 except:
     pass
 import numpy as np
@@ -25,7 +25,7 @@ from blendmodes.blend import blendLayers, BlendType
 import modules.sd_hijack
 from modules import devices, prompt_parser, masking, sd_samplers, lowvram, generation_parameters_copypaste, script_callbacks, extra_networks, sd_vae_approx, scripts # pylint: disable=unused-import
 from modules.sd_hijack import model_hijack
-from modules.shared import opts, cmd_opts, state # pylint: disable=unused-import
+from modules.shared import opts, cmd_opts, state, log # pylint: disable=unused-import
 import modules.shared as shared
 import modules.paths as paths
 import modules.face_restoration
@@ -35,13 +35,6 @@ import modules.sd_models as sd_models
 import modules.sd_vae as sd_vae
 import tomesd # pylint: disable=wrong-import-order
 
-
-# add a logger for the processing module
-logger = logging.getLogger(__name__)
-# manually set output level here since there is no option to do so yet through launch options
-# logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(name)s %(message)s')
-
-# some of those options should not be changed at all because they would break the model, so I removed them from options.
 opt_C = 4
 opt_f = 8
 
@@ -559,7 +552,7 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
 
         if (opts.token_merging or cmd_opts.token_merging) and not opts.token_merging_hr_only:
             sd_models.apply_token_merging(sd_model=p.sd_model, hr=False)
-            logger.debug('Token merging applied')
+            log.debug('Token merging applied')
 
         res = process_images_inner(p)
 
@@ -567,7 +560,7 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
         # undo model optimizations made by tomesd
         if opts.token_merging or cmd_opts.token_merging:
             tomesd.remove_patch(p.sd_model)
-            logger.debug('Token merging model optimizations removed')
+            log.debug('Token merging model optimizations removed')
 
         # restore opts to original state
         if p.override_settings_restore_afterwards:
@@ -778,47 +771,34 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                         image_without_cc = apply_overlay(image, p.paste_to, i, p.overlay_images)
                         images.save_image(image_without_cc, p.outpath_samples, "", seeds[i], prompts[i], opts.samples_format, info=infotext(n, i), p=p, suffix="-before-color-correction")
                     image = apply_color_correction(p.color_corrections[i], image)
-
                 image = apply_overlay(image, p.paste_to, i, p.overlay_images)
-
                 if opts.samples_save and not p.do_not_save_samples:
                     images.save_image(image, p.outpath_samples, "", seeds[i], prompts[i], opts.samples_format, info=infotext(n, i), p=p)
-
                 text = infotext(n, i)
                 infotexts.append(text)
                 if opts.enable_pnginfo:
                     image.info["parameters"] = text
                 output_images.append(image)
-
                 if hasattr(p, 'mask_for_overlay') and p.mask_for_overlay and any([opts.save_mask, opts.save_mask_composite, opts.return_mask, opts.return_mask_composite]):
                     image_mask = p.mask_for_overlay.convert('RGB')
                     image_mask_composite = Image.composite(image.convert('RGBA').convert('RGBa'), Image.new('RGBa', image.size), images.resize_image(2, p.mask_for_overlay, image.width, image.height).convert('L')).convert('RGBA')
-
                     if opts.save_mask:
                         images.save_image(image_mask, p.outpath_samples, "", seeds[i], prompts[i], opts.samples_format, info=infotext(n, i), p=p, suffix="-mask")
-
                     if opts.save_mask_composite:
                         images.save_image(image_mask_composite, p.outpath_samples, "", seeds[i], prompts[i], opts.samples_format, info=infotext(n, i), p=p, suffix="-mask-composite")
-
                     if opts.return_mask:
                         output_images.append(image_mask)
-
                     if opts.return_mask_composite:
                         output_images.append(image_mask_composite)
-
             del x_samples_ddim
-
             devices.torch_gc()
-
             state.nextjob()
 
         p.color_corrections = None
-
         index_of_first_image = 0
         unwanted_grid_because_of_img_count = len(output_images) < 2 and opts.grid_only_if_multiple
         if (opts.return_grid or opts.grid_save) and not p.do_not_save_grid and not unwanted_grid_because_of_img_count:
             grid = images.image_grid(output_images, p.batch_size)
-
             if opts.return_grid:
                 text = infotext()
                 infotexts.insert(0, text)
@@ -826,32 +806,25 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                     grid.info["parameters"] = text
                 output_images.insert(0, grid)
                 index_of_first_image = 1
-
             if opts.grid_save:
                 images.save_image(grid, p.outpath_grids, "grid", p.all_seeds[0], p.all_prompts[0], opts.grid_format, info=infotext(), short_filename=not opts.grid_extended_filename, p=p, grid=True)
 
     if not p.disable_extra_networks and extra_network_data:
         extra_networks.deactivate(p, extra_network_data)
-
     devices.torch_gc()
-
     res = Processed(p, output_images, p.all_seeds[0], infotext(), comments="".join(["\n\n" + x for x in comments]), subseed=p.all_subseeds[0], index_of_first_image=index_of_first_image, infotexts=infotexts)
-
     if p.scripts is not None:
         p.scripts.postprocess(p, res)
-
     return res
 
 
 def old_hires_fix_first_pass_dimensions(width, height):
     """old algorithm for auto-calculating first pass size"""
-
     desired_pixel_count = 512 * 512
     actual_pixel_count = width * height
     scale = math.sqrt(desired_pixel_count / actual_pixel_count)
     width = math.ceil(scale * width / 64) * 64
     height = math.ceil(scale * height / 64) * 64
-
     return width, height
 
 
@@ -869,13 +842,11 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
         self.hr_resize_y = hr_resize_y
         self.hr_upscale_to_x = hr_resize_x
         self.hr_upscale_to_y = hr_resize_y
-
         if firstphase_width != 0 or firstphase_height != 0:
             self.hr_upscale_to_x = self.width
             self.hr_upscale_to_y = self.height
             self.width = firstphase_width
             self.height = firstphase_height
-
         self.truncate_x = 0
         self.truncate_y = 0
         self.applied_old_hires_behavior_to = None
@@ -887,17 +858,14 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
                 self.hr_resize_y = self.height
                 self.hr_upscale_to_x = self.width
                 self.hr_upscale_to_y = self.height
-
                 self.width, self.height = old_hires_fix_first_pass_dimensions(self.width, self.height)
                 self.applied_old_hires_behavior_to = (self.width, self.height)
-
             if self.hr_resize_x == 0 and self.hr_resize_y == 0:
                 self.extra_generation_params["Hires upscale"] = self.hr_scale
                 self.hr_upscale_to_x = int(self.width * self.hr_scale)
                 self.hr_upscale_to_y = int(self.height * self.hr_scale)
             else:
                 self.extra_generation_params["Hires resize"] = f"{self.hr_resize_x}x{self.hr_resize_y}"
-
                 if self.hr_resize_y == 0:
                     self.hr_upscale_to_x = self.hr_resize_x
                     self.hr_upscale_to_y = self.hr_resize_x * self.height // self.width
@@ -909,17 +877,14 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
                     target_h = self.hr_resize_y
                     src_ratio = self.width / self.height
                     dst_ratio = self.hr_resize_x / self.hr_resize_y
-
                     if src_ratio < dst_ratio:
                         self.hr_upscale_to_x = self.hr_resize_x
                         self.hr_upscale_to_y = self.hr_resize_x * self.height // self.width
                     else:
                         self.hr_upscale_to_x = self.hr_resize_y * self.width // self.height
                         self.hr_upscale_to_y = self.hr_resize_y
-
                     self.truncate_x = (self.hr_upscale_to_x - target_w) // opt_f
                     self.truncate_y = (self.hr_upscale_to_y - target_h) // opt_f
-
             # special case: the user has chosen to do nothing
             if self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height:
                 self.enable_hr = False
@@ -927,53 +892,41 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
                 self.extra_generation_params.pop("Hires upscale", None)
                 self.extra_generation_params.pop("Hires resize", None)
                 return
-
             if not state.processing_has_refined_job_count:
                 if state.job_count == -1:
                     state.job_count = self.n_iter
                 state.job_count = state.job_count * 2
                 state.processing_has_refined_job_count = True
-
             if self.hr_second_pass_steps:
                 self.extra_generation_params["Hires steps"] = self.hr_second_pass_steps
-
             if self.hr_upscaler is not None:
                 self.extra_generation_params["Hires upscaler"] = self.hr_upscaler
 
     def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts):
         self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model)
-
         latent_scale_mode = shared.latent_upscale_modes.get(self.hr_upscaler, None) if self.hr_upscaler is not None else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "nearest")
         if self.enable_hr and latent_scale_mode is None:
             assert len([x for x in shared.sd_upscalers if x.name == self.hr_upscaler]) > 0, f"could not find upscaler named {self.hr_upscaler}"
-
         x = create_random_tensors([opt_C, self.height // opt_f, self.width // opt_f], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self)
         samples = self.sampler.sample(self, x, conditioning, unconditional_conditioning, image_conditioning=self.txt2img_image_conditioning(x))
-
         if not self.enable_hr:
             return samples
-
         target_width = self.hr_upscale_to_x
         target_height = self.hr_upscale_to_y
 
         def save_intermediate(image, index):
             """saves image before applying hires fix, if enabled in options; takes as an argument either an image or batch with latent space images"""
-
             if not opts.save or self.do_not_save_samples or not opts.save_images_before_highres_fix:
                 return
-
             if not isinstance(image, Image.Image):
                 image = sd_samplers.sample_to_image(image, index, approximation=0)
-
             info = create_infotext(self, self.all_prompts, self.all_seeds, self.all_subseeds, [], iteration=self.iteration, position_in_batch=index)
             images.save_image(image, self.outpath_samples, "", seeds[index], prompts[index], opts.samples_format, info=info, suffix="-before-highres-fix")
 
         if latent_scale_mode is not None:
             for i in range(samples.shape[0]):
                 save_intermediate(samples, i)
-
             samples = torch.nn.functional.interpolate(samples, size=(target_height // opt_f, target_width // opt_f), mode=latent_scale_mode["mode"], antialias=latent_scale_mode["antialias"])
-
             # Avoid making the inpainting conditioning unless necessary as
             # this does need some extra compute to decode / encode the image again.
             if getattr(self, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) < 1.0:
@@ -983,44 +936,32 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
         else:
             decoded_samples = decode_first_stage(self.sd_model, samples)
             lowres_samples = torch.clamp((decoded_samples + 1.0) / 2.0, min=0.0, max=1.0)
-
             batch_images = []
             for i, x_sample in enumerate(lowres_samples):
                 x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2)
                 x_sample = x_sample.astype(np.uint8)
                 image = Image.fromarray(x_sample)
-
                 save_intermediate(image, i)
-
                 image = images.resize_image(0, image, target_width, target_height, upscaler_name=self.hr_upscaler)
                 image = np.array(image).astype(np.float32) / 255.0
                 image = np.moveaxis(image, 2, 0)
                 batch_images.append(image)
-
             decoded_samples = torch.from_numpy(np.array(batch_images))
             decoded_samples = decoded_samples.to(shared.device)
             decoded_samples = 2. * decoded_samples - 1.
-
             samples = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(decoded_samples))
-
             image_conditioning = self.img2img_image_conditioning(decoded_samples, samples)
-
         shared.state.nextjob()
-
         img2img_sampler_name = self.sampler_name
         force_latent_upscaler = shared.opts.data.get('xyz_fallback_sampler')
         if self.sampler_name in ['PLMS'] or (force_latent_upscaler is not None and force_latent_upscaler != 'None'):
             img2img_sampler_name = force_latent_upscaler or shared.opts.fallback_sampler # PLMS does not support img2img, use fallback instead
         self.sampler = sd_samplers.create_sampler(img2img_sampler_name, self.sd_model)
-
         samples = samples[:, :, self.truncate_y//2:samples.shape[2]-(self.truncate_y+1)//2, self.truncate_x//2:samples.shape[3]-(self.truncate_x+1)//2]
-
         noise = create_random_tensors(samples.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=subseed_strength, p=self)
-
         # GC now before running the next img2img to prevent running out of memory
         x = None
         devices.torch_gc()
-
         # apply token merging optimizations from tomesd for high-res pass
         # check if hr_only so we are not redundantly patching
         if (cmd_opts.token_merging or opts.token_merging) and (opts.token_merging_hr_only or opts.token_merging_ratio_hr != opts.token_merging_ratio):
@@ -1028,13 +969,11 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
             if not opts.token_merging_hr_only:
                 # clean patch done by first pass. (clobbering the first patch might be fine? this might be excessive)
                 tomesd.remove_patch(self.sd_model)
-                logger.debug('Temporarily removed token merging optimizations in preparation for next pass')
+                log.debug('Temporarily removed token merging optimizations in preparation for next pass')
 
             sd_models.apply_token_merging(sd_model=self.sd_model, hr=True)
-            logger.debug('Applied token merging for high-res pass')
-
+            log.debug('Applied token merging for high-res pass')
         samples = self.sampler.sample_img2img(self, samples, noise, conditioning, unconditional_conditioning, steps=self.hr_second_pass_steps or self.steps, image_conditioning=image_conditioning)
-
         return samples
 
 
@@ -1043,7 +982,6 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
 
     def __init__(self, init_images: list = None, resize_mode: int = 0, denoising_strength: float = 0.75, image_cfg_scale: float = None, mask: Any = None, mask_blur: int = 4, inpainting_fill: int = 0, inpaint_full_res: bool = True, inpaint_full_res_padding: int = 0, inpainting_mask_invert: int = 0, initial_noise_multiplier: float = None, **kwargs):
         super().__init__(**kwargs)
-
         self.init_images = init_images
         self.resize_mode: int = resize_mode
         self.denoising_strength: float = denoising_strength
@@ -1115,30 +1053,23 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
             image = np.array(image).astype(np.float32) / 255.0
             image = np.moveaxis(image, 2, 0)
             imgs.append(image)
-
         if len(imgs) == 1:
             batch_images = np.expand_dims(imgs[0], axis=0).repeat(self.batch_size, axis=0)
             if self.overlay_images is not None:
                 self.overlay_images = self.overlay_images * self.batch_size
-
             if self.color_corrections is not None and len(self.color_corrections) == 1:
                 self.color_corrections = self.color_corrections * self.batch_size
-
         elif len(imgs) <= self.batch_size:
             self.batch_size = len(imgs)
             batch_images = np.array(imgs)
         else:
             raise RuntimeError(f"bad number of images passed: {len(imgs)}; expecting {self.batch_size} or less")
-
         image = torch.from_numpy(batch_images)
         image = 2. * image - 1.
         image = image.to(shared.device)
-
         self.init_latent = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(image))
-
         if self.resize_mode == 3:
             self.init_latent = torch.nn.functional.interpolate(self.init_latent, size=(self.height // opt_f, self.width // opt_f), mode="bilinear")
-
         if image_mask is not None:
             init_mask = latent_mask
             latmask = init_mask.convert('RGB').resize((self.init_latent.shape[3], self.init_latent.shape[2]))
@@ -1146,31 +1077,23 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
             latmask = latmask[0]
             latmask = np.around(latmask)
             latmask = np.tile(latmask[None], (4, 1, 1))
-
             self.mask = torch.asarray(1.0 - latmask).to(shared.device).type(self.sd_model.dtype)
             self.nmask = torch.asarray(latmask).to(shared.device).type(self.sd_model.dtype)
-
             # this needs to be fixed to be done in sample() using actual seeds for batches
             if self.inpainting_fill == 2:
                 self.init_latent = self.init_latent * self.mask + create_random_tensors(self.init_latent.shape[1:], all_seeds[0:self.init_latent.shape[0]]) * self.nmask
             elif self.inpainting_fill == 3:
                 self.init_latent = self.init_latent * self.mask
-
         self.image_conditioning = self.img2img_image_conditioning(image, self.init_latent, image_mask)
 
     def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts):
         x = create_random_tensors([opt_C, self.height // opt_f, self.width // opt_f], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self)
-
         if self.initial_noise_multiplier != 1.0:
             self.extra_generation_params["Noise multiplier"] = self.initial_noise_multiplier
             x *= self.initial_noise_multiplier
-
         samples = self.sampler.sample_img2img(self, self.init_latent, x, conditioning, unconditional_conditioning, image_conditioning=self.image_conditioning)
-
         if self.mask is not None:
             samples = samples * self.nmask + self.init_latent * self.mask
-
         del x
         devices.torch_gc()
-
         return samples

From 4dc5941912cc78bbe986a0c1fcdba6d7b194ab4d Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sun, 30 Apr 2023 21:01:49 -0400
Subject: [PATCH 65/69] fix embedding logging

---
 modules/shared.py                             |  1 -
 modules/textual_inversion/logging.py          |  5 +--
 .../textual_inversion/textual_inversion.py    | 45 ++-----------------
 webui.py                                      |  2 +-
 4 files changed, 7 insertions(+), 46 deletions(-)

diff --git a/modules/shared.py b/modules/shared.py
index b2a93542c..611d8af4d 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -352,7 +352,6 @@ options_templates.update(options_section(('training', "Training"), {
     "dataset_filename_word_regex": OptionInfo("", "Filename word regex"),
     "dataset_filename_join_string": OptionInfo(" ", "Filename join string"),
     "embeddings_templates_dir": OptionInfo(os.path.join(paths.script_path, 'train', 'templates'), "Embeddings train templates directory"),
-    "embeddings_train_log": OptionInfo(os.path.join(paths.script_path, 'train', 'log', 'train.csv'), "Embeddings train log file"),
     "training_image_repeats_per_epoch": OptionInfo(1, "Number of repeats for a single input image per epoch; used only for displaying epoch number", gr.Number, {"precision": 0}),
     "training_write_csv_every": OptionInfo(0, "Save an csv containing the loss to log directory every N steps, 0 to disable"),
     "training_enable_tensorboard": OptionInfo(False, "Enable tensorboard logging."),
diff --git a/modules/textual_inversion/logging.py b/modules/textual_inversion/logging.py
index d0e52beb7..b8440f656 100644
--- a/modules/textual_inversion/logging.py
+++ b/modules/textual_inversion/logging.py
@@ -16,8 +16,7 @@ def save_settings_to_file(log_directory, all_params):
     if all_params.get('preview_from_txt2img'):
         keys = keys | saved_params_previews
     params.update({k: v for k, v in all_params.items() if k in keys})
-    filename = 'settings.json'
-    fn = os.path.join(log_directory, filename)
+    filename = f"{params['embedding_name']}-{now.strftime('%Y-%m-%d_%H-%M-%S')}.json"
     with open(os.path.join(log_directory, filename), "w", encoding='utf-8') as file:
-        print(f'Training settings file: {fn}')
+        print(f'Training settings file: {os.path.join(log_directory, filename)}')
         json.dump(params, file, indent=2)
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index ab412f458..343e55539 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -4,7 +4,7 @@ import csv
 from collections import namedtuple
 import torch
 try:
-    import intel_extension_for_pytorch as ipex
+    import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
 except:
     pass
 import tqdm
@@ -283,20 +283,15 @@ def create_embedding(name, num_vectors_per_token, overwrite_old, init_text='*'):
 def write_loss(log_directory, filename, step, epoch_len, values):
     if shared.opts.training_write_csv_every == 0:
         return
-
-    if step % epoch_len != 0:
+    if step % shared.opts.training_write_csv_every != 0:
         return
     write_csv_header = False if os.path.exists(os.path.join(log_directory, filename)) else True
-
     with open(os.path.join(log_directory, filename), "a+", newline='', encoding='utf-8') as fout:
         csv_writer = csv.DictWriter(fout, fieldnames=["step", "epoch", "epoch_step", *(values.keys())])
-
         if write_csv_header:
             csv_writer.writeheader()
-
         epoch = (step - 1) // epoch_len
         epoch_step = (step - 1) % epoch_len
-
         csv_writer.writerow({
             "step": step,
             "epoch": epoch,
@@ -410,16 +405,11 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
         tensorboard_writer = tensorboard_setup(log_directory)
 
     pin_memory = shared.opts.pin_memory
-
     ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=embedding_name, model=shared.sd_model, cond_model=shared.sd_model.cond_stage_model, device=devices.device, template_file=template_file, batch_size=batch_size, gradient_step=gradient_step, shuffle_tags=shuffle_tags, tag_drop_out=tag_drop_out, latent_sampling_method=latent_sampling_method, varsize=varsize, use_weight=use_weight)
-
     if shared.opts.save_training_settings_to_txt:
         save_settings_to_file(log_directory, {**dict(model_name=checkpoint.model_name, model_hash=checkpoint.shorthash, num_of_dataset_images=len(ds), num_vectors_per_token=len(embedding.vec)), **locals()})
-
     latent_sampling_method = ds.latent_sampling_method
-
     dl = modules.textual_inversion.dataset.PersonalizedDataLoader(ds, latent_sampling_method=latent_sampling_method, batch_size=ds.batch_size, pin_memory=pin_memory)
-
     if unload:
         shared.parallel_processing_allowed = False
         shared.sd_model.first_stage_model.to(devices.cpu)
@@ -432,7 +422,6 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
             optimizer_saved_dict = torch.load(filename + '.optim', map_location='cpu')
             if embedding.checksum() == optimizer_saved_dict.get('hash', None):
                 optimizer_state_dict = optimizer_saved_dict.get('optimizer_state_dict', None)
-
         if optimizer_state_dict is not None:
             optimizer.load_state_dict(optimizer_state_dict)
             print("Loaded existing optimizer from checkpoint")
@@ -451,12 +440,10 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
     max_steps_per_epoch = len(ds) // batch_size - (len(ds) // batch_size) % gradient_step
     loss_step = 0
     _loss_step = 0 #internal
-
     last_saved_file = "<none>"
     last_saved_image = "<none>"
     forced_filename = "<none>"
     embedding_yet_to_be_embedded = False
-
     is_training_inpainting_model = shared.sd_model.model.conditioning_key in {'hybrid', 'concat'}
     img_c = None
 
@@ -478,7 +465,6 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
                     break
                 if shared.state.interrupted:
                     break
-
                 if clip_grad:
                     clip_grad_sched.step(embedding.step)
 
@@ -487,32 +473,26 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
                     if use_weight:
                         w = batch.weight.to(devices.device, non_blocking=pin_memory)
                     c = shared.sd_model.cond_stage_model(batch.cond_text)
-
                     if is_training_inpainting_model:
                         if img_c is None:
                             img_c = processing.txt2img_image_conditioning(shared.sd_model, c, training_width, training_height)
-
                         cond = {"c_concat": [img_c], "c_crossattn": [c]}
                     else:
                         cond = c
-
                     if use_weight:
                         loss = shared.sd_model.weighted_forward(x, cond, w)[0] / gradient_step
                         del w
                     else:
                         loss = shared.sd_model.forward(x, cond)[0] / gradient_step
                     del x
-
                     _loss_step += loss.item()
-                scaler.scale(loss).backward()
 
+                scaler.scale(loss).backward()
                 # go back until we reach gradient accumulation steps
                 if (j + 1) % gradient_step != 0:
                     continue
-
                 if clip_grad:
                     clip_grad(embedding.vec, clip_grad_sched.learn_rate)
-
                 scaler.step(optimizer)
                 scaler.update()
                 embedding.step += 1
@@ -520,9 +500,7 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
                 optimizer.zero_grad(set_to_none=True)
                 loss_step = _loss_step
                 _loss_step = 0
-
                 steps_done = embedding.step + 1
-
                 epoch_num = embedding.step // steps_per_epoch
 
                 description = f"Training textual inversion step {embedding.step} loss: {loss_step:.5f} lr: {scheduler.learn_rate:.5f}"
@@ -534,15 +512,11 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
                     save_embedding(embedding, optimizer, checkpoint, embedding_name_every, last_saved_file, remove_cached_checksum=True)
                     embedding_yet_to_be_embedded = True
 
-                write_loss(log_directory, shared.opts.embeddings_train_log, embedding.step, steps_per_epoch, {
-                    "loss": f"{loss_step:.7f}",
-                    "learn_rate": scheduler.learn_rate
-                })
+                write_loss(log_directory, f"{embedding_name}.csv", embedding.step, steps_per_epoch, { "loss": f"{loss_step:.7f}", "learn_rate": scheduler.learn_rate })
 
                 if images_dir is not None and steps_done % create_image_every == 0:
                     forced_filename = f'{embedding_name}-{steps_done}'
                     last_saved_image = os.path.join(images_dir, forced_filename)
-
                     shared.sd_model.first_stage_model.to(devices.device)
 
                     p = processing.StableDiffusionProcessingTxt2Img(
@@ -568,7 +542,6 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
                         p.height = training_height
 
                     preview_text = p.prompt
-
                     processed = processing.process_images(p)
                     image = processed.images[0] if len(processed.images) > 0 else None
 
@@ -577,35 +550,27 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
 
                     if image is not None:
                         shared.state.assign_current_image(image)
-
                         last_saved_image, _last_text_info = images.save_image(image, images_dir, "", p.seed, p.prompt, shared.opts.samples_format, processed.infotexts[0], p=p, forced_filename=forced_filename, save_to_dirs=False)
                         last_saved_image += f", prompt: {preview_text}"
-
                         if shared.opts.training_enable_tensorboard and shared.opts.training_tensorboard_save_images:
                             tensorboard_add_image(tensorboard_writer, f"Validation at epoch {epoch_num}", image, embedding.step)
 
                     if save_image_with_stored_embedding and os.path.exists(last_saved_file) and embedding_yet_to_be_embedded:
-
                         last_saved_image_chunks = os.path.join(images_embeds_dir, f'{embedding_name}-{steps_done}.png')
-
                         info = PngImagePlugin.PngInfo()
                         data = torch.load(last_saved_file)
                         info.add_text("sd-ti-embedding", embedding_to_b64(data))
                         title = f"<{data.get('name', '???')}>"
-
                         try:
                             vectorSize = list(data['string_to_param'].values())[0].shape[0]
                         except Exception:
                             vectorSize = '?'
-
                         checkpoint = sd_models.select_checkpoint()
                         footer_left = checkpoint.model_name
                         footer_mid = f'[{checkpoint.shorthash}]'
                         footer_right = f'{vectorSize}v {steps_done}s'
-
                         captioned_image = caption_image_overlay(image, title, footer_left, footer_mid, footer_right)
                         captioned_image = insert_image_data_embed(captioned_image, data)
-
                         captioned_image.save(last_saved_image_chunks, "PNG", pnginfo=info)
                         embedding_yet_to_be_embedded = False
 
@@ -613,7 +578,6 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
                     last_saved_image += f", prompt: {preview_text}"
 
                 shared.state.job_no = embedding.step
-
                 shared.state.textinfo = f"""
 <p>
 Loss: {loss_step:.7f}<br/>
@@ -633,7 +597,6 @@ Last saved image: {html.escape(last_saved_image)}<br/>
         shared.sd_model.first_stage_model.to(devices.device)
         shared.parallel_processing_allowed = old_parallel_processing_allowed
         sd_hijack_checkpoint.remove()
-
     return embedding, filename
 
 
diff --git a/webui.py b/webui.py
index 8d9e0dc2f..8954cfbfe 100644
--- a/webui.py
+++ b/webui.py
@@ -212,7 +212,7 @@ def start_ui():
     app, _local_url, _share_url = shared.demo.launch(
         share=cmd_opts.share,
         server_name=server_name,
-        server_port=cmd_opts.port,
+        server_port=cmd_opts.port if cmd_opts.port != 7860 else None,
         ssl_keyfile=cmd_opts.tls_keyfile,
         ssl_certfile=cmd_opts.tls_certfile,
         debug=False,

From 75b741f1199e526841b079c2e9b8ace99df2d1ad Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sun, 30 Apr 2023 23:06:32 -0400
Subject: [PATCH 66/69] fallback args

---
 extensions-builtin/sd-webui-controlnet | 2 +-
 installer.py                           | 2 +-
 modules/cmd_args.py                    | 6 ++++--
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index af4720780..14971922f 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit af4720780f10d912789cbd6db1fbc6d2f0afc533
+Subproject commit 14971922f0971095f0a9d0725d2027a26bf5dcb2
diff --git a/installer.py b/installer.py
index a7760f81e..56431ec20 100644
--- a/installer.py
+++ b/installer.py
@@ -20,7 +20,7 @@ class Dot(dict): # dot notation access to dictionary attributes
 
 
 log = logging.getLogger("sd")
-args = Dot({ 'debug': False, 'upgrade': False, 'skip_update': False, 'no_directml': False, 'skip_extensions': False, 'skip_requirements': False, 'reset': False })
+args = Dot({ 'debug': False, 'upgrade': False, 'no_directml': False, 'skip_update': False, 'skip_extensions': False, 'skip_requirements': False, 'skip_git': False, 'reset': False, 'use_ipex': False, 'experimental': False, 'test': False })
 quick_allowed = True
 errors = 0
 opts = {}
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index 4f74e5ad0..130354f37 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -76,6 +76,9 @@ def compatibility_args(opts, args):
     group.add_argument("--sub-quad-kv-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_kv_chunk_size)
     group.add_argument("--sub-quad-chunk-threshold", help=argparse.SUPPRESS, default=opts.sub_quad_chunk_threshold)
     group.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
+    group.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir)
+    group.add_argument("--lyco-dir", help=argparse.SUPPRESS, default=opts.lyco_dir)
+    group.add_argument("--use-ipex", action='store_true', help="Use Intel OneAPI XPU backend, default: %(default)s", default=False)
 
     opts.use_old_emphasis_implementation = False
     opts.use_old_karras_scheduler_sigmas = False
@@ -94,7 +97,6 @@ def compatibility_args(opts, args):
     opts.print_hypernet_extra = False
     opts.dimensions_and_batch_together = True
 
-    group.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir)
-    group.add_argument("--lyco-dir", help=argparse.SUPPRESS, default=opts.lyco_dir)
     args = parser.parse_args()
+
     return args

From f4256655b2c7266dad62795bf08f64ded6aa8d9a Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 1 May 2023 08:43:02 -0400
Subject: [PATCH 67/69] fix argparse

---
 extensions-builtin/sd-webui-controlnet |  2 +-
 modules/cmd_args.py                    | 31 ++++++++++++++------------
 modules/memmon.py                      |  5 +----
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index 14971922f..cfc37659a 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit 14971922f0971095f0a9d0725d2027a26bf5dcb2
+Subproject commit cfc37659aca364b37fc90943e039ceb2e7b6d8ba
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index 130354f37..e347c3532 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -5,17 +5,8 @@ from modules.paths_internal import data_path
 parser = argparse.ArgumentParser(description="SD.Next", conflict_handler='resolve', epilog='For other options see UI Settings page', prog='', add_help=True, formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=55, indent_increment=2, width=200))
 parser._optionals = parser.add_argument_group('Other options') # pylint: disable=protected-access
 group = parser.add_argument_group('Server options')
-# group.add_argument("--config", type=str, default=sd_default_config, help=argparse.SUPPRESS)
-
-group.add_argument("-f", action='store_true', help=argparse.SUPPRESS)  # allows running as root; implemented outside of webui
-group.add_argument("--ui-settings-file", type=str, help=argparse.SUPPRESS, default=os.path.join(data_path, 'config.json'))
-group.add_argument("--ui-config-file", type=str, help=argparse.SUPPRESS, default=os.path.join(data_path, 'ui-config.json'))
-group.add_argument("--hide-ui-dir-config", action='store_true', help=argparse.SUPPRESS, default=False)
-group.add_argument("--theme", type=str, help=argparse.SUPPRESS, default=None)
-group.add_argument("--disable-console-progressbars", action='store_true', help=argparse.SUPPRESS, default=True)
-group.add_argument("--disable-safe-unpickle", action='store_true', help=argparse.SUPPRESS, default=True)
-group.add_argument("--lowram", action='store_true', help=argparse.SUPPRESS)
 
+# main server args
 group.add_argument("--config", type=str, default=os.path.join(data_path, 'config.json'), help="Use specific configuration file, default: %(default)s")
 group.add_argument("--medvram", action='store_true', help="Split model stages and keep only active part in VRAM, default: %(default)s")
 group.add_argument("--lowvram", action='store_true', help="Split model components and keep only active part in VRAM, default: %(default)s")
@@ -45,9 +36,24 @@ group.add_argument("--no-hashing", action='store_true', help="Disable hashing of
 group.add_argument("--no-download", action='store_true', help="Disable download of default model, default: %(default)s", default=False)
 group.add_argument("--profile", action='store_true', help="Run profiler, default: %(default)s")
 group.add_argument("--disable-queue", action='store_true', help="Disable queues, default: %(default)s")
+group.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
+group.add_argument("--use-ipex", action='store_true', help="Use Intel OneAPI XPU backend, default: %(default)s", default=False)
+
+# removed args are added here as hidden in fixed format for compatbility reasons
+group.add_argument("-f", action='store_true', help=argparse.SUPPRESS)  # allows running as root; implemented outside of webui
+group.add_argument("--ui-settings-file", type=str, help=argparse.SUPPRESS, default=os.path.join(data_path, 'config.json'))
+group.add_argument("--ui-config-file", type=str, help=argparse.SUPPRESS, default=os.path.join(data_path, 'ui-config.json'))
+group.add_argument("--hide-ui-dir-config", action='store_true', help=argparse.SUPPRESS, default=False)
+group.add_argument("--theme", type=str, help=argparse.SUPPRESS, default=None)
+group.add_argument("--disable-console-progressbars", action='store_true', help=argparse.SUPPRESS, default=True)
+group.add_argument("--disable-safe-unpickle", action='store_true', help=argparse.SUPPRESS, default=True)
+group.add_argument("--lowram", action='store_true', help=argparse.SUPPRESS)
+group.add_argument("--disable-extension-access", default = False, action='store_true', help=argparse.SUPPRESS)
+group.add_argument("--api", help=argparse.SUPPRESS, default=True)
 
 
 def compatibility_args(opts, args):
+    # removed args that have been moved to opts are added here as hidden with default values as defined in opts
     group.add_argument("--ckpt-dir", type=str, help=argparse.SUPPRESS, default=opts.ckpt_dir)
     group.add_argument("--vae-dir", type=str, help=argparse.SUPPRESS, default=opts.vae_dir)
     group.add_argument("--embeddings-dir", type=str, help=argparse.SUPPRESS, default=opts.embeddings_dir)
@@ -62,7 +68,6 @@ def compatibility_args(opts, args):
     group.add_argument("--swinir-models-path", help=argparse.SUPPRESS, default=opts.swinir_models_path)
     group.add_argument("--ldsr-models-path", help=argparse.SUPPRESS, default=opts.ldsr_models_path)
     group.add_argument("--clip-models-path", type=str, help=argparse.SUPPRESS, default=opts.clip_models_path)
-    group.add_argument("--disable-extension-access", default = False, action='store_true', help=argparse.SUPPRESS)
     group.add_argument("--opt-channelslast", help=argparse.SUPPRESS, default=opts.opt_channelslast)
     group.add_argument("--xformers", default = (opts.cross_attention_optimization == "xFormers"), action='store_true', help=argparse.SUPPRESS)
     group.add_argument("--disable-nan-check", help=argparse.SUPPRESS, default=opts.disable_nan_check)
@@ -71,15 +76,13 @@ def compatibility_args(opts, args):
     group.add_argument("--no-half", help=argparse.SUPPRESS, default=opts.no_half)
     group.add_argument("--no-half-vae", help=argparse.SUPPRESS, default=opts.no_half_vae)
     group.add_argument("--precision", help=argparse.SUPPRESS, default=opts.precision)
-    group.add_argument("--api", help=argparse.SUPPRESS, default=True)
     group.add_argument("--sub-quad-q-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_q_chunk_size)
     group.add_argument("--sub-quad-kv-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_kv_chunk_size)
     group.add_argument("--sub-quad-chunk-threshold", help=argparse.SUPPRESS, default=opts.sub_quad_chunk_threshold)
-    group.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
     group.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir)
     group.add_argument("--lyco-dir", help=argparse.SUPPRESS, default=opts.lyco_dir)
-    group.add_argument("--use-ipex", action='store_true', help="Use Intel OneAPI XPU backend, default: %(default)s", default=False)
 
+    # removed opts are added here with fixed values for compatibility reasons
     opts.use_old_emphasis_implementation = False
     opts.use_old_karras_scheduler_sigmas = False
     opts.no_dpmpp_sde_batch_determinism = False
diff --git a/modules/memmon.py b/modules/memmon.py
index 3abc70ac3..77e5eb752 100644
--- a/modules/memmon.py
+++ b/modules/memmon.py
@@ -25,7 +25,7 @@ class MemUsageMonitor(threading.Thread):
         self.daemon = True
         self.run_flag = threading.Event()
         self.data = defaultdict(int)
-        if not torch.cuda.is_available() or not shared.cmd_opts.use_ipex:
+        if not torch.cuda.is_available():
             self.disabled = True
         else:
             if shared.cmd_opts.use_ipex:
@@ -35,7 +35,6 @@ class MemUsageMonitor(threading.Thread):
                 except Exception as e:  # AMD or whatever
                     print(f"Torch exception: {e}")
                     self.disabled = True
-
             else:
                 try:
                     self.cuda_mem_get_info()
@@ -96,7 +95,6 @@ class MemUsageMonitor(threading.Thread):
             free, total = self.cuda_mem_get_info()
             self.data["free"] = free
             self.data["total"] = total
-
             if shared.cmd_opts.use_ipex:
                 torch_stats = torch.xpu.memory_stats("xpu")
             else:
@@ -106,7 +104,6 @@ class MemUsageMonitor(threading.Thread):
             self.data["reserved"] = torch_stats["reserved_bytes.all.current"]
             self.data["reserved_peak"] = torch_stats["reserved_bytes.all.peak"]
             self.data["system_peak"] = total - self.data["min_free"]
-
         return self.data
 
     def stop(self):

From 22da90d4b8b1c5f582fb300b9b1e1d5808a02b60 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 1 May 2023 10:13:21 -0400
Subject: [PATCH 68/69] fix lora memory leak

---
 TODO.md                         | 1 +
 extensions-builtin/Lora/lora.py | 3 +++
 modules/sd_models.py            | 2 ++
 modules/shared.py               | 4 ++--
 4 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/TODO.md b/TODO.md
index 040fca0a2..556370885 100644
--- a/TODO.md
+++ b/TODO.md
@@ -20,6 +20,7 @@ Stuff to be added...
 - Auto-test `torch.layer_norm` for FP16
 - Monitor file changes by misbehaving extensions
 - Kitchen theme: <https://github.com/canisminor1990/sd-webui-kitchen-theme>
+- Lightbox improvements
 
 ## Investigate
 
diff --git a/extensions-builtin/Lora/lora.py b/extensions-builtin/Lora/lora.py
index 3cbd91646..ac3f3a8e5 100644
--- a/extensions-builtin/Lora/lora.py
+++ b/extensions-builtin/Lora/lora.py
@@ -260,6 +260,9 @@ def lora_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.Mu
     If not, restores orginal weights from backup and alters weights according to loras.
     """
 
+    if len(loaded_loras) == 0:
+        return
+
     lora_layer_name = getattr(self, 'lora_layer_name', None)
     if lora_layer_name is None:
         return
diff --git a/modules/sd_models.py b/modules/sd_models.py
index a7a9adeb9..332a19e65 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -236,6 +236,7 @@ def read_metadata_from_safetensors(filename):
 
 def read_state_dict(checkpoint_file, map_location=None): # pylint: disable=unused-argument
     try:
+        pl_sd = None
         with progress.open(checkpoint_file, 'rb', description=f'Loading weights: [cyan]{checkpoint_file}', auto_refresh=True) as f:
             _, extension = os.path.splitext(checkpoint_file)
             if 'v1-5-pruned-emaonly.safetensors' or 'vae-ft-mse-840000-ema-pruned.ckpt' in checkpoint_file:
@@ -251,6 +252,7 @@ def read_state_dict(checkpoint_file, map_location=None): # pylint: disable=unuse
                     buffer = io.BytesIO(f.read())
                     pl_sd = torch.load(buffer, map_location='cpu')
             sd = get_state_dict_from_checkpoint(pl_sd)
+        del pl_sd
     except Exception as e:
         errors.display(e, f'loading model: {checkpoint_file}')
         sd = None
diff --git a/modules/shared.py b/modules/shared.py
index 611d8af4d..77d5b3e14 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -5,12 +5,12 @@ import json
 import datetime
 import gradio as gr
 import tqdm
+from modules import errors, ui_components, shared_items, cmd_args
+from modules.paths_internal import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611
 import modules.interrogate
 import modules.memmon
 import modules.styles
 import modules.devices as devices
-from modules import errors, ui_components, shared_items, cmd_args
-from modules.paths_internal import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611
 import modules.paths_internal as paths
 from installer import log as central_logger # pylint: disable=E0611
 

From d4a748d758e1f3dd4bdf36c045fc30113043fe50 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 1 May 2023 12:25:35 -0400
Subject: [PATCH 69/69] update requirements

---
 TODO.md                                   | 1 -
 extensions-builtin/Lora/lora.py           | 3 ---
 extensions-builtin/a1111-sd-webui-lycoris | 2 +-
 extensions-builtin/sd-webui-controlnet    | 2 +-
 javascript/black-orange.css               | 7 ++++---
 modules/api/api.py                        | 4 ++--
 modules/cmd_args.py                       | 1 +
 requirements.txt                          | 6 +++---
 webui.py                                  | 1 +
 9 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/TODO.md b/TODO.md
index 556370885..a18ac0d0c 100644
--- a/TODO.md
+++ b/TODO.md
@@ -13,7 +13,6 @@ Stuff to be added...
 
 - Update `README.md`
 - Add Gradio theme maker
-- Transformers version
 - Create new GitHub hooks/actions for CI/CD  
 - Redo Extensions tab: <https://vladmandic.github.io/sd-extension-manager/pages/extensions.html>
 - Stream-load models as option for slow storage
diff --git a/extensions-builtin/Lora/lora.py b/extensions-builtin/Lora/lora.py
index ac3f3a8e5..3cbd91646 100644
--- a/extensions-builtin/Lora/lora.py
+++ b/extensions-builtin/Lora/lora.py
@@ -260,9 +260,6 @@ def lora_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.Mu
     If not, restores orginal weights from backup and alters weights according to loras.
     """
 
-    if len(loaded_loras) == 0:
-        return
-
     lora_layer_name = getattr(self, 'lora_layer_name', None)
     if lora_layer_name is None:
         return
diff --git a/extensions-builtin/a1111-sd-webui-lycoris b/extensions-builtin/a1111-sd-webui-lycoris
index ce584a0ff..4d74a7b88 160000
--- a/extensions-builtin/a1111-sd-webui-lycoris
+++ b/extensions-builtin/a1111-sd-webui-lycoris
@@ -1 +1 @@
-Subproject commit ce584a0ff863de98233ee135dcc17f2fb44703c3
+Subproject commit 4d74a7b889f91499dd85b9ccd7de58350b8da195
diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet
index cfc37659a..070d08f75 160000
--- a/extensions-builtin/sd-webui-controlnet
+++ b/extensions-builtin/sd-webui-controlnet
@@ -1 +1 @@
-Subproject commit cfc37659aca364b37fc90943e039ceb2e7b6d8ba
+Subproject commit 070d08f7524ae6d16973ee57b1e0bc6369d5388f
diff --git a/javascript/black-orange.css b/javascript/black-orange.css
index a055e2e51..703425ece 100644
--- a/javascript/black-orange.css
+++ b/javascript/black-orange.css
@@ -54,7 +54,7 @@ svg.feather.feather-image, .feather .feather-image { display: none }
 .py-6 { padding-bottom: 0; }
 .rounded-lg { border-radius: 0; }
 .tabs { background-color: black; }
-.gradio-button.tool { border-radius: 0; height: 2em; }
+.gradio-button.tool { border-radius: 0; height: 2.0em; }
 .block.token-counter span { background-color: #222 !important; box-shadow: 2px 2px 2px #111; border: none !important; border-radius: 0; font-size: 0.8rem; }
 .tab-nav { zoom: 130%; margin-bottom: 16px; border-bottom: 2px solid #CE6400 !important; padding-bottom: 2px; }
 .label-wrap { margin: 16px 0px 8px 0px; }
@@ -80,7 +80,7 @@ svg.feather.feather-image, .feather .feather-image { display: none }
 #lightboxModal { background-color: rgba(20, 20, 20, 0.8) }
 #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-left: -20px; margin-top: -2px; height: 2.4em; }
 #quicksettings > div, #quicksettings > fieldset { min-width: 26em; max-width: 26em; line-height: 2em; }
-#refresh_sd_model_checkpoint { height: 40px; margin-left: -14px; background: #333333; box-shadow: none; }
+#refresh_sd_model_checkpoint { height: 48px; margin-left: -14px; background: #333333; box-shadow: none; }
 #refresh_txt2img_styles, #refresh_img2img_styles, #open_folder_txt2img, #open_folder_img2img, #open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_res_switch_btn, #img2img_res_switch_btn, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h, #txt2img_tiling { display: none; }
 #save-animation { border-radius: 0 !important; margin-bottom: 16px; background-color: #111111; }
 #script_list { padding: 4px; margin-top: 20px; margin-bottom: 20px; }
@@ -103,7 +103,8 @@ svg.feather.feather-image, .feather .feather-image { display: none }
 #txt2img_tools, #img2img_tools { margin-top: 54px; scale: 120%; margin-left: 26px; }
 #txtimg_hr_finalres { max-width: 200px; }
 #pnginfo_html2_info { margin-top: -18px; background-color: var(--input-background-fill); padding: var(--input-padding) }
-#txt2img_extra_refresh, #txt2img_extra_close { height: 1.7em }
+#txt2img_extra_refresh, #txt2img_extra_close { height: 1.7em; }
+#extras_generate { margin-top: 8px; }
 
 /* custom elements overrides */
 #steps-animation, #controlnet { border-width: 0; }
diff --git a/modules/api/api.py b/modules/api/api.py
index 2912e2cf2..fbe64a037 100644
--- a/modules/api/api.py
+++ b/modules/api/api.py
@@ -13,8 +13,8 @@ import piexif
 import piexif.helper
 import uvicorn
 import gradio as gr
-from gradio.processing_utils import decode_base64_to_file
-# from gradio_client.utils import decode_base64_to_file
+# from gradio.processing_utils import decode_base64_to_file # gradio 3.23
+from gradio_client.utils import decode_base64_to_file # gradio 3.28
 
 from modules import errors, shared, sd_samplers, deepbooru, sd_hijack, images, scripts, ui, postprocessing
 from modules.api.models import * # pylint: disable=unused-wildcard-import, wildcard-import
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index e347c3532..80b11402d 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -31,6 +31,7 @@ group.add_argument("--cors-origins", type=str, help="Allowed CORS origins as com
 group.add_argument("--cors-regex", type=str, help="Allowed CORS origins as regular expression, default: %(default)s", default=None)
 group.add_argument("--tls-keyfile", type=str, help="Enable TLS and specify key file, default: %(default)s", default=None)
 group.add_argument("--tls-certfile", type=str, help="Enable TLS and specify cert file, default: %(default)s", default=None)
+group.add_argument("--tls-selfsign", action="store_true", help="Enable TLS with self-signed certificates, default: %(default)s", default=None)
 group.add_argument("--server-name", type=str, help="Sets hostname of server, default: %(default)s", default=None)
 group.add_argument("--no-hashing", action='store_true', help="Disable hashing of checkpoints, default: %(default)s", default=False)
 group.add_argument("--no-download", action='store_true', help="Disable download of default model, default: %(default)s", default=False)
diff --git a/requirements.txt b/requirements.txt
index 3f7db3419..e87a63e85 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -51,13 +51,13 @@ yapf
 scikit-image
 accelerate==0.18.0
 opencv-python==4.7.0.72
-diffusers==0.15.0
+diffusers==0.16.1
 einops==0.4.1
-gradio==3.23.0
+gradio==3.28.1
 numexpr==2.8.4
 pandas==1.5.3
 protobuf==3.20.3
 pytorch_lightning==1.9.4
-transformers==4.26.1
+transformers==4.28.1
 timm==0.6.13
 tomesd==0.1.2
diff --git a/webui.py b/webui.py
index 8954cfbfe..81e196bac 100644
--- a/webui.py
+++ b/webui.py
@@ -215,6 +215,7 @@ def start_ui():
         server_port=cmd_opts.port if cmd_opts.port != 7860 else None,
         ssl_keyfile=cmd_opts.tls_keyfile,
         ssl_certfile=cmd_opts.tls_certfile,
+        ssl_verify=False if cmd_opts.tls_selfsign else True,
         debug=False,
         auth=[tuple(cred.split(':')) for cred in gradio_auth_creds] if gradio_auth_creds else None,
         inbrowser=cmd_opts.autolaunch,