Merge branch 'dev' into pytorch-210

2026-09-19 17:24:32 +02:00 · 2023-10-10 16:29:12 +02:00
parent 0bf43ec2cf bfdafb6215
commit 4fbe883d88
37 changed files with 798 additions and 431 deletions
@@ -1,6 +1,6 @@
 # Change Log for SD.Next

-## Update for 2023-10-08
+## Update for 2023-10-09

 - Final strech of the DEV branch before merge to master  
 - Requires pending `diffusers==0.22.0`
@@ -133,7 +133,7 @@ or even free speedups and quality improvements (regardless of which workflows yo
  - to enable search, make sure all models have set hash values  
    *Models -> Valida -> Calculate hashes*  
 - **LoRA**
-  - new unified LoRA handler for all LoRA types (lora, lyco, loha, lokr, locon, etc.)  
+  - new unified LoRA handler for all LoRA types (lora, lyco, loha, lokr, locon, ia3, etc.)  
    applies to both original and diffusers backend  
    thanks @AI-Casanova for diffusers port  
  - for *backend:original*, separate lyco handler has been removed  
@@ -11,12 +11,9 @@ N/A
 Stuff to be added, in no particular order...

 - Diffusers:
-  - Add Lyco support
  - Add ControlNet
-  - Fix DeepFloyd IF model
  - Add unCLIP model
  - Add Training support
-  - Add long prompts
 - Technical debt:
  - Port **A1111** stuff
  - Port `p.all_hr_prompts`
@@ -24,15 +21,11 @@ Stuff to be added, in no particular order...
 - Non-technical:
  - Update Wiki
  - Rename repo: **automatic** -> **sdnext**
-  - [Localization](https://app.transifex.com/signup/open-source/)
 - New Minor
  - Prompt padding for positive/negative
-  - Add EN provider for VAEs
-  - Built-in `motd`-style notifications
 - New Major
  - Profile manager (for `config.json` and `ui-config.json`)
  - Multi-user support
-  - Add [SAG](https://huggingface.co/docs/diffusers/v0.19.3/en/api/pipelines/self_attention_guidance), [SAG](https://github.com/ashen-sensored/sd_webui_SAG)
  - Image phash and hdash using `imagehash`
  - Model merge using `git-rebasin`
  - Enable refiner-style workflow for `ldm` backend
@@ -48,6 +41,8 @@ Stuff to be added, in no particular order...
  - New inpainting canvas controls (move from backend to purely frontend)
  - New image browser (move from backend to purely frontend)
  - Change workflows from static/legacy to steps-based
+- Video processing
+

 ## Investigate

@@ -71,6 +66,8 @@ Tech that can be integrated as part of the core workflow...
 - [QuickEmbedding](https://github.com/ethansmith2000/QuickEmbedding)
 - [DragGAN](https://github.com/XingangPan/DragGAN)
 - [LamaCleaner](https://github.com/Sanster/lama-cleaner)
+- [SAG](https://huggingface.co/docs/diffusers/v0.19.3/en/api/pipelines/self_attention_guidance), [SAG](https://github.com/ashen-sensored/sd_webui_SAG)
+- [Localization](https://app.transifex.com/signup/open-source/)
 - `TensorRT`

 ## Random
@@ -1,19 +1,31 @@
+import time
 import networks
+import lora_patches
 from modules import extra_networks, shared


 class ExtraNetworkLora(extra_networks.ExtraNetwork):
+
    def __init__(self):
        super().__init__('lora')
+        self.active = False
        self.errors = {}
+        networks.originals = lora_patches.LoraPatches()
+
        """mapping of network names to the number of errors the network had during operation"""

    def activate(self, p, params_list):
+        t0 = time.time()
        additional = shared.opts.sd_lora
        self.errors.clear()
        if additional != "None" and additional in networks.available_networks and not any(x for x in params_list if x.items[0] == additional):
            p.all_prompts = [x + f"<lora:{additional}:{shared.opts.extra_networks_default_multiplier}>" for x in p.all_prompts]
            params_list.append(extra_networks.ExtraNetworkParams(items=[additional, shared.opts.extra_networks_default_multiplier]))
+        if len(params_list) > 0:
+            self.active = True
+            networks.originals.apply() # apply patches
+            if networks.debug:
+                shared.log.debug("LoRA activate")
        names = []
        te_multipliers = []
        unet_multipliers = []
@@ -30,7 +42,9 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
            te_multipliers.append(te_multiplier)
            unet_multipliers.append(unet_multiplier)
            dyn_dims.append(dyn_dim)
+        t1 = time.time()
        networks.load_networks(names, te_multipliers, unet_multipliers, dyn_dims)
+        t2 = time.time()
        if shared.opts.lora_add_hashes_to_infotext:
            network_hashes = []
            for item in networks.loaded_networks:
@@ -44,8 +58,18 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
                network_hashes.append(f"{alias}: {shorthash}")
            if network_hashes:
                p.extra_generation_params["Lora hashes"] = ", ".join(network_hashes)
+        if len(names) > 0:
+            shared.log.info(f'Applying LoRA: {names} patch={t1-t0:.2f}s load={t2-t1:.2f}s')
+        elif self.active:
+            self.active = False

    def deactivate(self, p):
+        if not self.active and getattr(networks, "originals", None ) is not None:
+            networks.originals.undo() # remove patches
+            if networks.debug:
+                shared.log.debug("LoRA deactivate")
+        if self.active and networks.debug:
+            shared.log.debug(f"LoRA end: load={networks.timer['load']:.2f}s apply={networks.timer['apply']:.2f}s restore={networks.timer['restore']:.2f}s")
        if self.errors:
            p.comment("Networks with errors: " + ", ".join(f"{k} ({v})" for k, v in self.errors.items()))
            for k, v in self.errors.items():
@@ -0,0 +1,208 @@
+from typing import Dict
+import re
+import bisect
+from modules import shared
+
+
+suffix_conversion = {
+    "attentions": {},
+    "resnets": {
+        "conv1": "in_layers_2",
+        "conv2": "out_layers_3",
+        "norm1": "in_layers_0",
+        "norm2": "out_layers_0",
+        "time_emb_proj": "emb_layers_1",
+        "conv_shortcut": "skip_connection",
+    }
+}
+re_digits = re.compile(r"\d+")
+re_x_proj = re.compile(r"(.*)_([qkv]_proj)$")
+re_compiled = {}
+
+
+def make_unet_conversion_map() -> Dict[str, str]:
+    unet_conversion_map_layer = []
+
+    for i in range(3):  # num_blocks is 3 in sdxl
+        # loop over downblocks/upblocks
+        for j in range(2):
+            # loop over resnets/attentions for downblocks
+            hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}."
+            sd_down_res_prefix = f"input_blocks.{3 * i + j + 1}.0."
+            unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix))
+            if i < 3:
+                # no attention layers in down_blocks.3
+                hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}."
+                sd_down_atn_prefix = f"input_blocks.{3 * i + j + 1}.1."
+                unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix))
+
+        for j in range(3):
+            # loop over resnets/attentions for upblocks
+            hf_up_res_prefix = f"up_blocks.{i}.resnets.{j}."
+            sd_up_res_prefix = f"output_blocks.{3 * i + j}.0."
+            unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix))
+            # if i > 0: commentout for sdxl
+            # no attention layers in up_blocks.0
+            hf_up_atn_prefix = f"up_blocks.{i}.attentions.{j}."
+            sd_up_atn_prefix = f"output_blocks.{3 * i + j}.1."
+            unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix))
+
+        if i < 3:
+            # no downsample in down_blocks.3
+            hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv."
+            sd_downsample_prefix = f"input_blocks.{3 * (i + 1)}.0.op."
+            unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix))
+            # no upsample in up_blocks.3
+            hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0."
+            sd_upsample_prefix = f"output_blocks.{3 * i + 2}.{2}."  # change for sdxl
+            unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix))
+
+    hf_mid_atn_prefix = "mid_block.attentions.0."
+    sd_mid_atn_prefix = "middle_block.1."
+    unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix))
+
+    for j in range(2):
+        hf_mid_res_prefix = f"mid_block.resnets.{j}."
+        sd_mid_res_prefix = f"middle_block.{2 * j}."
+        unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix))
+
+    unet_conversion_map_resnet = [
+        # (stable-diffusion, HF Diffusers)
+        ("in_layers.0.", "norm1."),
+        ("in_layers.2.", "conv1."),
+        ("out_layers.0.", "norm2."),
+        ("out_layers.3.", "conv2."),
+        ("emb_layers.1.", "time_emb_proj."),
+        ("skip_connection.", "conv_shortcut."),
+    ]
+
+    unet_conversion_map = []
+    for sd, hf in unet_conversion_map_layer:
+        if "resnets" in hf:
+            for sd_res, hf_res in unet_conversion_map_resnet:
+                unet_conversion_map.append((sd + sd_res, hf + hf_res))
+        else:
+            unet_conversion_map.append((sd, hf))
+
+    for j in range(2):
+        hf_time_embed_prefix = f"time_embedding.linear_{j + 1}."
+        sd_time_embed_prefix = f"time_embed.{j * 2}."
+        unet_conversion_map.append((sd_time_embed_prefix, hf_time_embed_prefix))
+
+    for j in range(2):
+        hf_label_embed_prefix = f"add_embedding.linear_{j + 1}."
+        sd_label_embed_prefix = f"label_emb.0.{j * 2}."
+        unet_conversion_map.append((sd_label_embed_prefix, hf_label_embed_prefix))
+
+    unet_conversion_map.append(("input_blocks.0.0.", "conv_in."))
+    unet_conversion_map.append(("out.0.", "conv_norm_out."))
+    unet_conversion_map.append(("out.2.", "conv_out."))
+
+    sd_hf_conversion_map = {sd.replace(".", "_")[:-1]: hf.replace(".", "_")[:-1] for sd, hf in unet_conversion_map}
+    return sd_hf_conversion_map
+
+
+class KeyConvert:
+    def __init__(self):
+        if shared.backend == shared.Backend.ORIGINAL:
+            self.converter = self.original
+            self.is_sd2 = 'model_transformer_resblocks' in shared.sd_model.network_layer_mapping
+
+        else:
+            self.converter = self.diffusers
+            self.is_sdxl = True if shared.sd_model_type == "sdxl" else False
+            self.UNET_CONVERSION_MAP = make_unet_conversion_map() if self.is_sdxl else None
+            self.LORA_PREFIX_UNET = "lora_unet"
+            self.LORA_PREFIX_TEXT_ENCODER = "lora_te"
+            # SDXL: must starts with LORA_PREFIX_TEXT_ENCODER
+            self.LORA_PREFIX_TEXT_ENCODER1 = "lora_te1"
+            self.LORA_PREFIX_TEXT_ENCODER2 = "lora_te2"
+
+    def original(self, key):
+        key = convert_diffusers_name_to_compvis(key, self.is_sd2)
+        sd_module = shared.sd_model.network_layer_mapping.get(key, None)
+        if sd_module is None:
+            m = re_x_proj.match(key)
+            if m:
+                sd_module = shared.sd_model.network_layer_mapping.get(m.group(1), None)
+        # SDXL loras seem to already have correct compvis keys, so only need to replace "lora_unet" with "diffusion_model"
+        if sd_module is None and "lora_unet" in key:
+            key = key.replace("lora_unet", "diffusion_model")
+            sd_module = shared.sd_model.network_layer_mapping.get(key, None)
+        elif sd_module is None and "lora_te1_text_model" in key:
+            key = key.replace("lora_te1_text_model", "0_transformer_text_model")
+            sd_module = shared.sd_model.network_layer_mapping.get(key, None)
+            # some SD1 Loras also have correct compvis keys
+            if sd_module is None:
+                key = key.replace("lora_te1_text_model", "transformer_text_model")
+                sd_module = shared.sd_model.network_layer_mapping.get(key, None)
+        return key, sd_module
+
+    def diffusers(self, key):
+        if self.is_sdxl:
+            map_keys = list(self.UNET_CONVERSION_MAP.keys())  # prefix of U-Net modules
+            map_keys.sort()
+            search_key = key.replace(self.LORA_PREFIX_UNET + "_", "").replace(self.LORA_PREFIX_TEXT_ENCODER1 + "_",
+                                                                              "").replace(
+                self.LORA_PREFIX_TEXT_ENCODER2 + "_", "")
+            position = bisect.bisect_right(map_keys, search_key)
+            map_key = map_keys[position - 1]
+            if search_key.startswith(map_key):
+                key = key.replace(map_key, self.UNET_CONVERSION_MAP[map_key]) # pylint: disable=unsubscriptable-object
+        sd_module = shared.sd_model.network_layer_mapping.get(key, None)
+        return key, sd_module
+
+    def __call__(self, key):
+        return self.converter(key)
+
+
+def convert_diffusers_name_to_compvis(key, is_sd2):
+    def match(match_list, regex_text):
+        regex = re_compiled.get(regex_text)
+        if regex is None:
+            regex = re.compile(regex_text)
+            re_compiled[regex_text] = regex
+        r = re.match(regex, key)
+        if not r:
+            return False
+        match_list.clear()
+        match_list.extend([int(x) if re.match(re_digits, x) else x for x in r.groups()])
+        return True
+
+    m = []
+    if match(m, r"lora_unet_conv_in(.*)"):
+        return f'diffusion_model_input_blocks_0_0{m[0]}'
+    if match(m, r"lora_unet_conv_out(.*)"):
+        return f'diffusion_model_out_2{m[0]}'
+    if match(m, r"lora_unet_time_embedding_linear_(\d+)(.*)"):
+        return f"diffusion_model_time_embed_{m[0] * 2 - 2}{m[1]}"
+    if match(m, r"lora_unet_down_blocks_(\d+)_(attentions|resnets)_(\d+)_(.+)"):
+        suffix = suffix_conversion.get(m[1], {}).get(m[3], m[3])
+        return f"diffusion_model_input_blocks_{1 + m[0] * 3 + m[2]}_{1 if m[1] == 'attentions' else 0}_{suffix}"
+    if match(m, r"lora_unet_mid_block_(attentions|resnets)_(\d+)_(.+)"):
+        suffix = suffix_conversion.get(m[0], {}).get(m[2], m[2])
+        return f"diffusion_model_middle_block_{1 if m[0] == 'attentions' else m[1] * 2}_{suffix}"
+    if match(m, r"lora_unet_up_blocks_(\d+)_(attentions|resnets)_(\d+)_(.+)"):
+        suffix = suffix_conversion.get(m[1], {}).get(m[3], m[3])
+        return f"diffusion_model_output_blocks_{m[0] * 3 + m[2]}_{1 if m[1] == 'attentions' else 0}_{suffix}"
+    if match(m, r"lora_unet_down_blocks_(\d+)_downsamplers_0_conv"):
+        return f"diffusion_model_input_blocks_{3 + m[0] * 3}_0_op"
+    if match(m, r"lora_unet_up_blocks_(\d+)_upsamplers_0_conv"):
+        return f"diffusion_model_output_blocks_{2 + m[0] * 3}_{2 if m[0]>0 else 1}_conv"
+    if match(m, r"lora_te_text_model_encoder_layers_(\d+)_(.+)"):
+        if is_sd2:
+            if 'mlp_fc1' in m[1]:
+                return f"model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc1', 'mlp_c_fc')}"
+            elif 'mlp_fc2' in m[1]:
+                return f"model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc2', 'mlp_c_proj')}"
+            else:
+                return f"model_transformer_resblocks_{m[0]}_{m[1].replace('self_attn', 'attn')}"
+        return f"transformer_text_model_encoder_layers_{m[0]}_{m[1]}"
+    if match(m, r"lora_te2_text_model_encoder_layers_(\d+)_(.+)"):
+        if 'mlp_fc1' in m[1]:
+            return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc1', 'mlp_c_fc')}"
+        elif 'mlp_fc2' in m[1]:
+            return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc2', 'mlp_c_proj')}"
+        else:
+            return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('self_attn', 'attn')}"
+    return key
@@ -5,6 +5,21 @@ from modules import patches

 class LoraPatches:
    def __init__(self):
+        self.active = False
+        self.Linear_forward = None
+        self.Linear_load_state_dict = None
+        self.Conv2d_forward = None
+        self.Conv2d_load_state_dict = None
+        self.GroupNorm_forward = None
+        self.GroupNorm_load_state_dict = None
+        self.LayerNorm_forward = None
+        self.LayerNorm_load_state_dict = None
+        self.MultiheadAttention_forward = None
+        self.MultiheadAttention_load_state_dict = None
+
+    def apply(self):
+        if self.active:
+            return
        self.Linear_forward = patches.patch(__name__, torch.nn.Linear, 'forward', networks.network_Linear_forward)
        self.Linear_load_state_dict = patches.patch(__name__, torch.nn.Linear, '_load_from_state_dict', networks.network_Linear_load_state_dict)
        self.Conv2d_forward = patches.patch(__name__, torch.nn.Conv2d, 'forward', networks.network_Conv2d_forward)
@@ -15,8 +30,14 @@ class LoraPatches:
        self.LayerNorm_load_state_dict = patches.patch(__name__, torch.nn.LayerNorm, '_load_from_state_dict', networks.network_LayerNorm_load_state_dict)
        self.MultiheadAttention_forward = patches.patch(__name__, torch.nn.MultiheadAttention, 'forward', networks.network_MultiheadAttention_forward)
        self.MultiheadAttention_load_state_dict = patches.patch(__name__, torch.nn.MultiheadAttention, '_load_from_state_dict', networks.network_MultiheadAttention_load_state_dict)
+        networks.timer['load'] = 0
+        networks.timer['apply'] = 0
+        networks.timer['restore'] = 0
+        self.active = True

    def undo(self):
+        if not self.active:
+            return
        self.Linear_forward = patches.undo(__name__, torch.nn.Linear, 'forward') # pylint: disable=E1128
        self.Linear_load_state_dict = patches.undo(__name__, torch.nn.Linear, '_load_from_state_dict') # pylint: disable=E1128
        self.Conv2d_forward = patches.undo(__name__, torch.nn.Conv2d, 'forward') # pylint: disable=E1128
@@ -27,3 +48,5 @@ class LoraPatches:
        self.LayerNorm_load_state_dict = patches.undo(__name__, torch.nn.LayerNorm, '_load_from_state_dict') # pylint: disable=E1128
        self.MultiheadAttention_forward = patches.undo(__name__, torch.nn.MultiheadAttention, 'forward') # pylint: disable=E1128
        self.MultiheadAttention_load_state_dict = patches.undo(__name__, torch.nn.MultiheadAttention, '_load_from_state_dict') # pylint: disable=E1128
+        patches.originals.pop(__name__, None)
+        self.active = False
@@ -75,10 +75,7 @@ class NetworkOnDisk:

    def get_alias(self):
        import networks
-        if shared.opts.lora_preferred_name == "Filename" or self.alias.lower() in networks.forbidden_network_aliases:
-            return self.name
-        else:
-            return self.alias
+        return self.name if shared.opts.lora_preferred_name == "filename" or self.alias.lower() in networks.forbidden_network_aliases else self.alias


 class Network:  # LoraModule
@@ -1,8 +1,7 @@
-from typing import Dict, Union
-import logging
+from typing import Union
 import os
 import re
-import bisect
+import time
 import lora_patches
 import network
 import network_lora
@@ -11,11 +10,24 @@ import network_ia3
 import network_lokr
 import network_full
 import network_norm
+import lora_convert
 import torch
+import diffusers.models.lora
 from modules import shared, devices, sd_models, errors, scripts, sd_hijack
-import diffusers.models.lora as diffusers_lora


+debug = os.environ.get('SD_LORA_DEBUG', None)
+originals: lora_patches.LoraPatches = None
+extra_network_lora = None
+available_networks = {}
+available_network_aliases = {}
+loaded_networks = []
+timer = { 'load': 0, 'apply': 0, 'restore': 0 }
+# networks_in_memory = {}
+lora_cache = {}
+available_network_hash_lookup = {}
+forbidden_network_aliases = {}
+re_network_name = re.compile(r"(.*)\s*\([0-9a-fA-F]+\)")
 module_types = [
    network_lora.ModuleTypeLora(),
    network_hada.ModuleTypeHada(),
@@ -26,215 +38,6 @@ module_types = [
 ]


-re_digits = re.compile(r"\d+")
-re_x_proj = re.compile(r"(.*)_([qkv]_proj)$")
-re_compiled = {}
-
-suffix_conversion = {
-    "attentions": {},
-    "resnets": {
-        "conv1": "in_layers_2",
-        "conv2": "out_layers_3",
-        "norm1": "in_layers_0",
-        "norm2": "out_layers_0",
-        "time_emb_proj": "emb_layers_1",
-        "conv_shortcut": "skip_connection",
-    }
-}
-
-
-def make_unet_conversion_map() -> Dict[str, str]:
-    unet_conversion_map_layer = []
-
-    for i in range(3):  # num_blocks is 3 in sdxl
-        # loop over downblocks/upblocks
-        for j in range(2):
-            # loop over resnets/attentions for downblocks
-            hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}."
-            sd_down_res_prefix = f"input_blocks.{3 * i + j + 1}.0."
-            unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix))
-
-            if i < 3:
-                # no attention layers in down_blocks.3
-                hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}."
-                sd_down_atn_prefix = f"input_blocks.{3 * i + j + 1}.1."
-                unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix))
-
-        for j in range(3):
-            # loop over resnets/attentions for upblocks
-            hf_up_res_prefix = f"up_blocks.{i}.resnets.{j}."
-            sd_up_res_prefix = f"output_blocks.{3 * i + j}.0."
-            unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix))
-
-            # if i > 0: commentout for sdxl
-            # no attention layers in up_blocks.0
-            hf_up_atn_prefix = f"up_blocks.{i}.attentions.{j}."
-            sd_up_atn_prefix = f"output_blocks.{3 * i + j}.1."
-            unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix))
-
-        if i < 3:
-            # no downsample in down_blocks.3
-            hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv."
-            sd_downsample_prefix = f"input_blocks.{3 * (i + 1)}.0.op."
-            unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix))
-
-            # no upsample in up_blocks.3
-            hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0."
-            sd_upsample_prefix = f"output_blocks.{3 * i + 2}.{2}."  # change for sdxl
-            unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix))
-
-    hf_mid_atn_prefix = "mid_block.attentions.0."
-    sd_mid_atn_prefix = "middle_block.1."
-    unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix))
-
-    for j in range(2):
-        hf_mid_res_prefix = f"mid_block.resnets.{j}."
-        sd_mid_res_prefix = f"middle_block.{2 * j}."
-        unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix))
-
-    unet_conversion_map_resnet = [
-        # (stable-diffusion, HF Diffusers)
-        ("in_layers.0.", "norm1."),
-        ("in_layers.2.", "conv1."),
-        ("out_layers.0.", "norm2."),
-        ("out_layers.3.", "conv2."),
-        ("emb_layers.1.", "time_emb_proj."),
-        ("skip_connection.", "conv_shortcut."),
-    ]
-
-    unet_conversion_map = []
-    for sd, hf in unet_conversion_map_layer:
-        if "resnets" in hf:
-            for sd_res, hf_res in unet_conversion_map_resnet:
-                unet_conversion_map.append((sd + sd_res, hf + hf_res))
-        else:
-            unet_conversion_map.append((sd, hf))
-
-    for j in range(2):
-        hf_time_embed_prefix = f"time_embedding.linear_{j + 1}."
-        sd_time_embed_prefix = f"time_embed.{j * 2}."
-        unet_conversion_map.append((sd_time_embed_prefix, hf_time_embed_prefix))
-
-    for j in range(2):
-        hf_label_embed_prefix = f"add_embedding.linear_{j + 1}."
-        sd_label_embed_prefix = f"label_emb.0.{j * 2}."
-        unet_conversion_map.append((sd_label_embed_prefix, hf_label_embed_prefix))
-
-    unet_conversion_map.append(("input_blocks.0.0.", "conv_in."))
-    unet_conversion_map.append(("out.0.", "conv_norm_out."))
-    unet_conversion_map.append(("out.2.", "conv_out."))
-
-    sd_hf_conversion_map = {sd.replace(".", "_")[:-1]: hf.replace(".", "_")[:-1] for sd, hf in unet_conversion_map}
-    return sd_hf_conversion_map
-
-
-class KeyConvert:
-    def __init__(self):
-        if shared.backend == shared.Backend.ORIGINAL:
-            self.converter = self.original
-            self.is_sd2 = 'model_transformer_resblocks' in shared.sd_model.network_layer_mapping
-
-        else:
-            self.converter = self.diffusers
-            self.is_sdxl = True if shared.sd_model_type == "sdxl" else False
-            self.UNET_CONVERSION_MAP = make_unet_conversion_map() if self.is_sdxl else None
-            self.LORA_PREFIX_UNET = "lora_unet"
-            self.LORA_PREFIX_TEXT_ENCODER = "lora_te"
-
-            # SDXL: must starts with LORA_PREFIX_TEXT_ENCODER
-            self.LORA_PREFIX_TEXT_ENCODER1 = "lora_te1"
-            self.LORA_PREFIX_TEXT_ENCODER2 = "lora_te2"
-
-    def original(self, key):
-        key = convert_diffusers_name_to_compvis(key, self.is_sd2)
-        sd_module = shared.sd_model.network_layer_mapping.get(key, None)
-        if sd_module is None:
-            m = re_x_proj.match(key)
-            if m:
-                sd_module = shared.sd_model.network_layer_mapping.get(m.group(1), None)
-        # SDXL loras seem to already have correct compvis keys, so only need to replace "lora_unet" with "diffusion_model"
-        if sd_module is None and "lora_unet" in key:
-            key = key.replace("lora_unet", "diffusion_model")
-            sd_module = shared.sd_model.network_layer_mapping.get(key, None)
-        elif sd_module is None and "lora_te1_text_model" in key:
-            key = key.replace("lora_te1_text_model", "0_transformer_text_model")
-            sd_module = shared.sd_model.network_layer_mapping.get(key, None)
-            # some SD1 Loras also have correct compvis keys
-            if sd_module is None:
-                key = key.replace("lora_te1_text_model", "transformer_text_model")
-                sd_module = shared.sd_model.network_layer_mapping.get(key, None)
-        return key, sd_module
-
-    def diffusers(self, key):
-        if self.is_sdxl:
-            map_keys = list(self.UNET_CONVERSION_MAP.keys())  # prefix of U-Net modules
-            map_keys.sort()
-            search_key = key.replace(self.LORA_PREFIX_UNET + "_", "").replace(self.LORA_PREFIX_TEXT_ENCODER1 + "_",
-                                                                              "").replace(
-                self.LORA_PREFIX_TEXT_ENCODER2 + "_", "")
-            position = bisect.bisect_right(map_keys, search_key)
-            map_key = map_keys[position - 1]
-            if search_key.startswith(map_key):
-                key = key.replace(map_key, self.UNET_CONVERSION_MAP[map_key])
-        sd_module = shared.sd_model.network_layer_mapping.get(key, None)
-        return key, sd_module
-
-    def __call__(self, key):
-        return self.converter(key)
-
-
-def convert_diffusers_name_to_compvis(key, is_sd2):
-    def match(match_list, regex_text):
-        regex = re_compiled.get(regex_text)
-        if regex is None:
-            regex = re.compile(regex_text)
-            re_compiled[regex_text] = regex
-        r = re.match(regex, key)
-        if not r:
-            return False
-        match_list.clear()
-        match_list.extend([int(x) if re.match(re_digits, x) else x for x in r.groups()])
-        return True
-
-    m = []
-    if match(m, r"lora_unet_conv_in(.*)"):
-        return f'diffusion_model_input_blocks_0_0{m[0]}'
-    if match(m, r"lora_unet_conv_out(.*)"):
-        return f'diffusion_model_out_2{m[0]}'
-    if match(m, r"lora_unet_time_embedding_linear_(\d+)(.*)"):
-        return f"diffusion_model_time_embed_{m[0] * 2 - 2}{m[1]}"
-    if match(m, r"lora_unet_down_blocks_(\d+)_(attentions|resnets)_(\d+)_(.+)"):
-        suffix = suffix_conversion.get(m[1], {}).get(m[3], m[3])
-        return f"diffusion_model_input_blocks_{1 + m[0] * 3 + m[2]}_{1 if m[1] == 'attentions' else 0}_{suffix}"
-    if match(m, r"lora_unet_mid_block_(attentions|resnets)_(\d+)_(.+)"):
-        suffix = suffix_conversion.get(m[0], {}).get(m[2], m[2])
-        return f"diffusion_model_middle_block_{1 if m[0] == 'attentions' else m[1] * 2}_{suffix}"
-    if match(m, r"lora_unet_up_blocks_(\d+)_(attentions|resnets)_(\d+)_(.+)"):
-        suffix = suffix_conversion.get(m[1], {}).get(m[3], m[3])
-        return f"diffusion_model_output_blocks_{m[0] * 3 + m[2]}_{1 if m[1] == 'attentions' else 0}_{suffix}"
-    if match(m, r"lora_unet_down_blocks_(\d+)_downsamplers_0_conv"):
-        return f"diffusion_model_input_blocks_{3 + m[0] * 3}_0_op"
-    if match(m, r"lora_unet_up_blocks_(\d+)_upsamplers_0_conv"):
-        return f"diffusion_model_output_blocks_{2 + m[0] * 3}_{2 if m[0]>0 else 1}_conv"
-    if match(m, r"lora_te_text_model_encoder_layers_(\d+)_(.+)"):
-        if is_sd2:
-            if 'mlp_fc1' in m[1]:
-                return f"model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc1', 'mlp_c_fc')}"
-            elif 'mlp_fc2' in m[1]:
-                return f"model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc2', 'mlp_c_proj')}"
-            else:
-                return f"model_transformer_resblocks_{m[0]}_{m[1].replace('self_attn', 'attn')}"
-        return f"transformer_text_model_encoder_layers_{m[0]}_{m[1]}"
-    if match(m, r"lora_te2_text_model_encoder_layers_(\d+)_(.+)"):
-        if 'mlp_fc1' in m[1]:
-            return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc1', 'mlp_c_fc')}"
-        elif 'mlp_fc2' in m[1]:
-            return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc2', 'mlp_c_proj')}"
-        else:
-            return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('self_attn', 'attn')}"
-    return key
-
-
 def assign_network_names_to_compvis_modules(sd_model):
    """
    if shared.sd_model.is_sdxl:
@@ -282,14 +85,19 @@ def assign_network_names_to_compvis_modules(sd_model):


 def load_network(name, network_on_disk):
+    t0 = time.time()
+    cached = lora_cache.get(name, None)
+    if debug:
+        shared.log.debug(f'LoRA load: name={name} file={network_on_disk.filename} {"cached" if cached else ""}')
+    if cached is not None:
+        return cached
    net = network.Network(name, network_on_disk)
    net.mtime = os.path.getmtime(network_on_disk.filename)
    sd = sd_models.read_state_dict(network_on_disk.filename)
-    # this should not be needed but is here as an emergency fix for an unknown error people are experiencing in 1.2.0
-    assign_network_names_to_compvis_modules(shared.sd_model)
+    assign_network_names_to_compvis_modules(shared.sd_model) # this should not be needed but is here as an emergency fix for an unknown error people are experiencing in 1.2.0
    keys_failed_to_match = {}
    matched_networks = {}
-    convert = KeyConvert()
+    convert = lora_convert.KeyConvert()
    for key_network, weight in sd.items():
        key_network_without_network_parts, network_part = key_network.split(".", 1)
        key, sd_module = convert(key_network_without_network_parts)
@@ -309,28 +117,21 @@ def load_network(name, network_on_disk):
            raise AssertionError(f"Could not find a module type (out of {', '.join([x.__class__.__name__ for x in module_types])}) that would accept those keys: {', '.join(weights.w)}")
        net.modules[key] = net_module
    if keys_failed_to_match:
-        logging.debug(f"Network {network_on_disk.filename} didn't match keys: {keys_failed_to_match}")
+        shared.log.warning(f"LoRA unmatched keys: file={network_on_disk.filename} keys={len(keys_failed_to_match)}")
+        if debug:
+            shared.log.debug(f"LoRA unmatched keys: file={network_on_disk.filename} keys={keys_failed_to_match}")
+    lora_cache[name] = net
+    t1 = time.time()
+    timer['load'] += t1 - t0
    return net

-def purge_networks_from_memory():
-    while len(networks_in_memory) > shared.opts.lora_in_memory_limit and len(networks_in_memory) > 0:
-        name = next(iter(networks_in_memory))
-        networks_in_memory.pop(name, None)
-    devices.torch_gc()
-

 def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None):
-    already_loaded = {}
-    for net in loaded_networks:
-        if net.name in names:
-            already_loaded[net.name] = net
-    loaded_networks.clear()
    networks_on_disk = [available_network_aliases.get(name, None) for name in names]
    if any(x is None for x in networks_on_disk):
        list_available_networks()
        networks_on_disk = [available_network_aliases.get(name, None) for name in names]
    failed_to_load_networks = []
-
    recompile_model = False
    if shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx":
        if len(names) == len(shared.compiled_model_state.lora_model):
@@ -346,25 +147,21 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No
        shared.opts.cuda_compile = False
        sd_models.reload_model_weights(op='model')
        shared.opts.cuda_compile = True
-
+    loaded_networks.clear()
    for i, (network_on_disk, name) in enumerate(zip(networks_on_disk, names)):
-        net = already_loaded.get(name, None)
        if network_on_disk is not None:
-            if net is None:
-                net = networks_in_memory.get(name)
-            if net is None or os.path.getmtime(network_on_disk.filename) > net.mtime:
-                try:
-                    net = load_network(name, network_on_disk)
-                    networks_in_memory.pop(name, None)
-                    networks_in_memory[name] = net
-                except Exception as e:
-                    errors.display(e, f"loading network {network_on_disk.filename}")
-                    continue
+            try:
+                net = load_network(name, network_on_disk)
+            except Exception as e:
+                shared.log.error(f"LoRA load failed: file={network_on_disk.filename}")
+                if debug:
+                    errors.display(e, f"LoRA load failed file={network_on_disk.filename}")
+                continue
            net.mentioned_name = name
            network_on_disk.read_hash()
        if net is None:
            failed_to_load_networks.append(name)
-            logging.info(f"Couldn't find network with name {name}")
+            shared.log.error(f"LoRA unknown network: file={network_on_disk.filename} network={name}")
            continue
        net.te_multiplier = te_multipliers[i] if te_multipliers else 1.0
        net.unet_multiplier = unet_multipliers[i] if unet_multipliers else 1.0
@@ -372,25 +169,33 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No
        loaded_networks.append(net)
    if failed_to_load_networks:
        sd_hijack.model_hijack.comments.append("Networks not found: " + ", ".join(failed_to_load_networks))
-    purge_networks_from_memory()
+
+    while len(lora_cache) > shared.opts.lora_in_memory_limit:
+        name = next(iter(lora_cache))
+        lora_cache.pop(name, None)
+    if len(loaded_networks) > 0 and debug:
+        shared.log.debug(f'LoRA loaded={len(loaded_networks)} cache={list(lora_cache)}')
+    devices.torch_gc()

    if recompile_model:
-        shared.log.info("Networks: Recompiling model")
+        shared.log.info("LoRA recompiling model")
        sd_models.compile_diffusers(shared.sd_model)


-def network_restore_weights_from_backup(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, torch.nn.MultiheadAttention, diffusers_lora.LoRACompatibleLinear, diffusers_lora.LoRACompatibleConv]):
+def network_restore_weights_from_backup(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, torch.nn.MultiheadAttention, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv]):
+    t0 = time.time()
    weights_backup = getattr(self, "network_weights_backup", None)
    bias_backup = getattr(self, "network_bias_backup", None)
    if weights_backup is None and bias_backup is None:
        return
+    # if debug:
+    #     shared.log.debug('LoRA restore weights')
    if weights_backup is not None:
        if isinstance(self, torch.nn.MultiheadAttention):
            self.in_proj_weight.copy_(weights_backup[0])
            self.out_proj.weight.copy_(weights_backup[1])
        else:
            self.weight.copy_(weights_backup)
-
    if bias_backup is not None:
        if isinstance(self, torch.nn.MultiheadAttention):
            self.out_proj.bias.copy_(bias_backup)
@@ -401,9 +206,11 @@ def network_restore_weights_from_backup(self: Union[torch.nn.Conv2d, torch.nn.Li
            self.out_proj.bias = None
        else:
            self.bias = None
+    t1 = time.time()
+    timer['restore'] += t1 - t0


-def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, torch.nn.MultiheadAttention, diffusers_lora.LoRACompatibleLinear, diffusers_lora.LoRACompatibleConv]):
+def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, torch.nn.MultiheadAttention, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv]):
    """
    Applies the currently selected set of networks to the weights of torch layer self.
    If weights already have this particular set of networks applied, does nothing.
@@ -412,6 +219,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
    network_layer_name = getattr(self, 'network_layer_name', None)
    if network_layer_name is None:
        return
+    t0 = time.time()
    current_names = getattr(self, "network_current_names", ())
    wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks)
    weights_backup = getattr(self, "network_weights_backup", None)
@@ -432,6 +240,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
        else:
            bias_backup = None
        self.network_bias_backup = bias_backup
+
    if current_names != wanted_names:
        network_restore_weights_from_backup(self)
        for net in loaded_networks:
@@ -442,7 +251,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
                        updown, ex_bias = module.calc_updown(self.weight)
                        if len(self.weight.shape) == 4 and self.weight.shape[1] == 9:
                            # inpainting model. zero pad updown to make channel[1]  4 to 9
-                            updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5))
+                            updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable
                        self.weight += updown
                        if ex_bias is not None and hasattr(self, 'bias'):
                            if self.bias is None:
@@ -450,7 +259,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
                            else:
                                self.bias += ex_bias
                except RuntimeError as e:
-                    logging.debug(f"Network {net.name} layer {network_layer_name}: {e}")
+                    if debug:
+                        shared.log.debug(f"LoRA apply weight network={net.name} layer={network_layer_name} {e}")
                    extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1
                continue
            module_q = net.modules.get(network_layer_name + "_q_proj", None)
@@ -473,14 +283,17 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
                        else:
                            self.out_proj.bias += ex_bias
                except RuntimeError as e:
-                    logging.debug(f"Network {net.name} layer {network_layer_name}: {e}")
+                    if debug:
+                        shared.log.debug(f"LoRA network={net.name} layer={network_layer_name} {e}")
                    extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1
                continue
            if module is None:
                continue
-            logging.debug(f"Network {net.name} layer {network_layer_name}: couldn't find supported operation")
+            shared.log.warning(f"LoRA network={net.name} layer={network_layer_name} unsupported operation")
            extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1
        self.network_current_names = wanted_names
+    t1 = time.time()
+    timer['apply'] += t1 - t0


 def network_forward(module, input, original_forward): # pylint: disable=W0622
@@ -590,8 +403,6 @@ def list_available_networks():
        available_network_aliases[name] = entry
        available_network_aliases[entry.alias] = entry

-re_network_name = re.compile(r"(.*)\s*\([0-9a-fA-F]+\)")
-

 def infotext_pasted(infotext, params): # pylint: disable=W0613
    if "AddNet Module 1" in [x[1] for x in scripts.scripts_txt2img.infotext_fields]:
@@ -615,12 +426,4 @@ def infotext_pasted(infotext, params): # pylint: disable=W0613
        params["Prompt"] += "\n" + "".join(added)


-originals: lora_patches.LoraPatches = None
-extra_network_lora = None
-available_networks = {}
-available_network_aliases = {}
-loaded_networks = []
-networks_in_memory = {}
-available_network_hash_lookup = {}
-forbidden_network_aliases = {}
 list_available_networks()
@@ -4,14 +4,14 @@ from fastapi import FastAPI
 import network
 import networks
 import lora  # noqa:F401 # pylint: disable=unused-import
-import lora_patches
+# import lora_patches
 import extra_networks_lora
 import ui_extra_networks_lora
 from modules import script_callbacks, ui_extra_networks, extra_networks, shared


-def unload():
-    networks.originals.undo()
+# def unload():
+#     networks.originals.undo()


 def before_ui():
@@ -21,25 +21,18 @@ def before_ui():
    # extra_networks.register_extra_network_alias(networks.extra_network_lora, "lyco")


-networks.originals = lora_patches.LoraPatches()
+# networks.originals = lora_patches.LoraPatches()
 script_callbacks.on_model_loaded(networks.assign_network_names_to_compvis_modules)
-script_callbacks.on_script_unloaded(unload)
+# script_callbacks.on_script_unloaded(unload)
 script_callbacks.on_before_ui(before_ui)
 script_callbacks.on_infotext_pasted(networks.infotext_pasted)


 shared.options_templates.update(shared.options_section(('extra_networks', "Extra Networks"), {
-    "sd_lora": shared.OptionInfo("None", "Add network to prompt", gr.Dropdown, lambda: {"choices": ["None", *networks.available_networks], "visible": False}, refresh=networks.list_available_networks),
-    "lora_preferred_name": shared.OptionInfo("Alias from file", "When adding to prompt, refer to Lora by", gr.Radio, {"choices": ["Alias from file", "Filename"]}),
-    "lora_add_hashes_to_infotext": shared.OptionInfo(True, "Add Lora hashes to infotext"),
+    # "sd_lora": shared.OptionInfo("None", "Add network to prompt", gr.Dropdown, lambda: {"choices": ["None", *networks.available_networks], "visible": False}, refresh=networks.list_available_networks),
+    "sd_lora": shared.OptionInfo("None", "Add network to prompt", gr.Dropdown, {"choices": ["None"]}),
    # "lora_show_all": shared.OptionInfo(False, "Always show all networks on the Lora page").info("otherwise, those detected as for incompatible version of Stable Diffusion will be hidden"),
    # "lora_hide_unknown_for_versions": shared.OptionInfo([], "Hide networks of unknown versions for model versions", gr.CheckboxGroup, {"choices": ["SD1", "SD2", "SDXL"]}),
-    "lora_in_memory_limit": shared.OptionInfo(0, "Lora in-memory cache", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}),
-}))
-
-
-shared.options_templates.update(shared.options_section(('compatibility', "Compatibility"), {
-    "lora_functional": shared.OptionInfo(False, "Lora/Networks: use old method that takes longer when you have multiple Loras active and produces same results as kohya-ss/sd-webui-additional-networks extension"),
 }))


@@ -85,4 +78,3 @@ def infotext_pasted(infotext, d): # pylint: disable=unused-argument


 script_callbacks.on_infotext_pasted(infotext_pasted)
-shared.opts.onchange("lora_in_memory_limit", networks.purge_networks_from_memory)
@@ -18,6 +18,7 @@ class Dot(dict): # dot notation access to dictionary attributes
    __delattr__ = dict.__delitem__


+version = None
 log = logging.getLogger("sd")
 log_file = os.path.join(os.path.dirname(__file__), 'sdnext.log')
 log_rolled = False
@@ -177,16 +178,16 @@ def installed(package, friendly: str = None):
                spec = pkg_resources.working_set.by_key.get(p[0].replace('_', '-'), None) # check name variations
            ok = ok and spec is not None
            if ok:
-                version = pkg_resources.get_distribution(p[0]).version
-                # log.debug(f"Package version found: {p[0]} {version}")
+                package_version = pkg_resources.get_distribution(p[0]).version
+                # log.debug(f"Package version found: {p[0]} {package_version}")
                if len(p) > 1:
-                    exact = version == p[1]
+                    exact = package_version == p[1]
                    ok = ok and (exact or args.experimental)
                    if not exact:
                        if args.experimental:
-                            log.warning(f"Package allowing experimental: {p[0]} {version} required {p[1]}")
+                            log.warning(f"Package allowing experimental: {p[0]} {package_version} required {p[1]}")
                        else:
-                            log.warning(f"Package wrong version: {p[0]} {version} required {p[1]}")
+                            log.warning(f"Package wrong version: {p[0]} {package_version} required {p[1]}")
            else:
                log.debug(f"Package version not found: {p[0]}")
        return ok
@@ -468,8 +469,8 @@ def check_torch():
                try:
                    if args.use_directml and allow_directml:
                        import torch_directml # pylint: disable=import-error
-                        version = pkg_resources.get_distribution("torch-directml")
-                        log.info(f'Torch backend: DirectML ({version})')
+                        dml_ver = pkg_resources.get_distribution("torch-directml")
+                        log.info(f'Torch backend: DirectML ({dml_ver})')
                        for i in range(0, torch_directml.device_count()):
                            log.info(f'Torch detected GPU: {torch_directml.device_name(i)}')
                except Exception:
@@ -562,24 +563,18 @@ def install_repositories():
    log.info('Verifying repositories')
    os.makedirs(os.path.join(os.path.dirname(__file__), 'repositories'), exist_ok=True)
    stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git")
-    # stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf")
    stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', None)
    clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit)
    taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git")
-    # taming_transformers_commit = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "3ba01b241669f5ade541ce990f7650a3b8f65318")
    taming_transformers_commit = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', None)
    clone(taming_transformers_repo, d('taming-transformers'), taming_transformers_commit)
    k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git')
-    # k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919")
-    # k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', 'ab527a9')
-    k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', 'f4a74f1ec906cb62916f58288ec73ef0330ba446')
+    k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', '0455157')
    clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit)
    codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git')
-    # codeformer_commit = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af")
    codeformer_commit = os.environ.get('CODEFORMER_COMMIT_HASH', "7a584fd")
    clone(codeformer_repo, d('CodeFormer'), codeformer_commit)
    blip_repo = os.environ.get('BLIP_REPO', 'https://github.com/salesforce/BLIP.git')
-    # blip_commit = os.environ.get('BLIP_COMMIT_HASH', "48211a1594f1321b00f14c9f7a5b4813144b2fb9")
    blip_commit = os.environ.get('BLIP_COMMIT_HASH', None)
    clone(blip_repo, d('BLIP'), blip_commit)
    if args.profile:
@@ -786,8 +781,12 @@ def check_extensions():


 def get_version():
-    version = None
+    global version # pylint: disable=global-statement
    if version is None:
+        try:
+            subprocess.run('git config log.showsignature false', stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True, check=True)
+        except Exception:
+            pass
        try:
            res = subprocess.run('git log --pretty=format:"%h %ad" -1 --date=short', stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True, check=True)
            ver = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else '  '
@@ -874,6 +873,7 @@ def check_timestamp():
        return True
    ok = True
    setup_time = -1
+    version_time = -1
    with open(log_file, 'r', encoding='utf8') as f:
        lines = f.readlines()
        for line in lines:
@@ -883,8 +883,6 @@ def check_timestamp():
        version_time = int(git('log -1 --pretty=format:"%at"'))
    except Exception as e:
        log.error(f'Error getting local repository version: {e}')
-        if not args.ignore:
-            sys.exit(1)
    log.debug(f'Repository update time: {time.ctime(int(version_time))}')
    if setup_time == -1:
        return False
@@ -1,6 +1,6 @@
 let currentWidth = null;
 let currentHeight = null;
-let arFrameTimeout = setTimeout(() => {}, 0);
+let arFrameTimeout = null;

 function dimensionChange(e, is_width, is_height) {
  if (is_width) currentWidth = e.target.value * 1.0;
@@ -42,10 +42,8 @@ function dimensionChange(e, is_width, is_height) {
    arPreviewRect.style.width = `${arRectWidth}px`;
    arPreviewRect.style.height = `${arRectHeight}px`;

-    clearTimeout(arFrameTimeout);
-    arFrameTimeout = setTimeout(() => {
-      arPreviewRect.style.display = 'none';
-    }, 2000);
+    if (arFrameTimeout) clearTimeout(arFrameTimeout);
+    arFrameTimeout = setTimeout(() => { arPreviewRect.style.display = 'none'; }, 2000);
    arPreviewRect.style.display = 'block';
  }
 }
@@ -34,11 +34,16 @@ body, button, input, select, textarea { font-family: var(--font);}
 button { font-size: 1.2rem; max-width: 400px; }
 img { background-color: var(--background-color); }
 input[type=checkbox] { background-color: transparent !important; }
-input[type=range] { height: var(--line-sm); appearance: none; margin-top: 0; min-width: 160px; background-color: var(--background-color); width: 100%; background: transparent; }
-input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: var(--input-background-fill); border-radius: var(--radius-lg); border: 0px solid #222222; }
-input[type=range]::-moz-range-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: var(--input-background-fill); border-radius: var(--radius-lg); border: 0px solid #222222; }
-input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: var(--radius-lg); background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; }
-input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: var(--radius-lg); background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; }
+input[type=range] { height: var(--line-sm) !important; appearance: none !important; margin-top: 0 !important; min-width: 160px !important;
+  background-color: var(--background-color) !important; width: 100% !important; background: transparent !important; }
+input[type=range]::-webkit-slider-runnable-track { width: 100% !important; height: var(--line-sm) !important; cursor: pointer !important; box-shadow: 2px 2px 3px #111111 !important;
+  background: var(--input-background-fill) !important; border-radius: var(--radius-lg) !important; border: 0px solid #222222 !important; }
+input[type=range]::-moz-range-track { width: 100% !important; height: var(--line-sm) !important; cursor: pointer !important; box-shadow: 2px 2px 3px #111111 !important; background:
+  var(--input-background-fill) !important; border-radius: var(--radius-lg) !important; border: 0px solid #222222 !important; }
+input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111 !important; border: 0px solid #000000 !important; height: var(--line-sm) !important; width: var(--line-sm) !important;
+  border-radius: var(--radius-lg) !important; background: var(--highlight-color) !important; cursor: pointer !important; appearance: none !important; margin-top: 0px !important; }
+input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111 !important; border: 0px solid #000000 !important; height: var(--line-sm) !important; width: var(--line-sm) !important;
+  border-radius: var(--radius-lg) !important; background: var(--highlight-color) !important; cursor: pointer !important; appearance: none !important; margin-top: 0px !important; }
 ::-webkit-scrollbar { width: 12px; }
 ::-webkit-scrollbar-track { background: #333333; }
 ::-webkit-scrollbar-thumb { background-color: var(--highlight-color); border-radius: var(--radius-lg); border-width: 0; box-shadow: 2px 2px 3px #111111; }
@@ -40,11 +40,16 @@ html { font-size: var(--font-size); }
 body, button, input, select, textarea { font-family: var(--font);}
 button { font-size: 1.2rem; max-width: 400px; }
 img { background-color: var(--background-color); }
-input[type=range] { height: var(--line-sm); appearance: none; margin-top: 0; min-width: 160px; background-color: var(--background-color); width: 100%; background: transparent; }
-input[type=range]::-webkit-slider-runnable-track { width: 100%; height: var(--line-sm); cursor: pointer; box-shadow: 2px 2px 3px #111111; background: var(--input-background-fill); border-radius: var(--radius-lg); border: 0px solid #222222; }
-input[type=range]::-moz-range-track { width: 100%; height: var(--line-sm); cursor: pointer; box-shadow: 2px 2px 3px #111111; background: var(--input-background-fill); border-radius: var(--radius-lg); border: 0px solid #222222; }
-input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: var(--line-sm); width: var(--line-sm); border-radius: var(--radius-lg); background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; }
-input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: var(--line-sm); width: var(--line-sm); border-radius: var(--radius-lg); background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; }
+input[type=range] { height: var(--line-sm) !important; appearance: none !important; margin-top: 0 !important; min-width: 160px !important;
+  background-color: var(--background-color) !important; width: 100% !important; background: transparent !important; }
+input[type=range]::-webkit-slider-runnable-track { width: 100% !important; height: var(--line-sm) !important; cursor: pointer !important; box-shadow: 2px 2px 3px #111111 !important;
+  background: var(--input-background-fill) !important; border-radius: var(--radius-lg) !important; border: 0px solid #222222 !important; }
+input[type=range]::-moz-range-track { width: 100% !important; height: var(--line-sm) !important; cursor: pointer !important; box-shadow: 2px 2px 3px #111111 !important; background:
+  var(--input-background-fill) !important; border-radius: var(--radius-lg) !important; border: 0px solid #222222 !important; }
+input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111 !important; border: 0px solid #000000 !important; height: var(--line-sm) !important; width: var(--line-sm) !important;
+  border-radius: var(--radius-lg) !important; background: var(--highlight-color) !important; cursor: pointer !important; appearance: none !important; margin-top: 0px !important; }
+input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111 !important; border: 0px solid #000000 !important; height: var(--line-sm) !important; width: var(--line-sm) !important;
+  border-radius: var(--radius-lg) !important; background: var(--highlight-color) !important; cursor: pointer !important; appearance: none !important; margin-top: 0px !important; }
 ::-webkit-scrollbar { width: 12px; }
 ::-webkit-scrollbar-track { background: #333333; }
 ::-webkit-scrollbar-thumb { background-color: var(--highlight-color); border-radius: var(--radius-lg); border-width: 0; box-shadow: 2px 2px 3px #111111; }
@@ -40,11 +40,16 @@ html { font-size: var(--font-size); }
 body, button, input, select, textarea { font-family: var(--font);}
 button { font-size: 1.2rem; max-width: 400px; --button-large-radius: var(--radius-lg) }
 img { background-color: var(--background-color); }
-input[type=range] { height: var(--line-sm); appearance: none; margin-top: 0; min-width: 160px; background-color: var(--background-color); width: 100%; background: transparent; }
-input[type=range]::-webkit-slider-runnable-track { width: 100%; height: var(--line-sm); cursor: pointer; box-shadow: 2px 2px 3px #111111; background: var(--input-background-fill); border-radius: var(--radius-lg); border: 0px solid #222222; }
-input[type=range]::-moz-range-track { width: 100%; height: var(--line-sm); cursor: pointer; box-shadow: 2px 2px 3px #111111; background: var(--input-background-fill); border-radius: var(--radius-lg); border: 0px solid #222222; }
-input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: var(--line-sm); width: var(--line-sm); border-radius: var(--radius-lg); background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; }
-input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: var(--line-sm); width: var(--line-sm); border-radius: var(--radius-lg); background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; }
+input[type=range] { height: var(--line-sm) !important; appearance: none !important; margin-top: 0 !important; min-width: 160px !important;
+  background-color: var(--background-color) !important; width: 100% !important; background: transparent !important; }
+input[type=range]::-webkit-slider-runnable-track { width: 100% !important; height: var(--line-sm) !important; cursor: pointer !important; box-shadow: 2px 2px 3px #111111 !important;
+  background: var(--input-background-fill) !important; border-radius: var(--radius-lg) !important; border: 0px solid #222222 !important; }
+input[type=range]::-moz-range-track { width: 100% !important; height: var(--line-sm) !important; cursor: pointer !important; box-shadow: 2px 2px 3px #111111 !important; background:
+  var(--input-background-fill) !important; border-radius: var(--radius-lg) !important; border: 0px solid #222222 !important; }
+input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111 !important; border: 0px solid #000000 !important; height: var(--line-sm) !important; width: var(--line-sm) !important;
+  border-radius: var(--radius-lg) !important; background: var(--highlight-color) !important; cursor: pointer !important; appearance: none !important; margin-top: 0px !important; }
+input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111 !important; border: 0px solid #000000 !important; height: var(--line-sm) !important; width: var(--line-sm) !important;
+  border-radius: var(--radius-lg) !important; background: var(--highlight-color) !important; cursor: pointer !important; appearance: none !important; margin-top: 0px !important; }
 ::-webkit-scrollbar { width: 12px; }
 ::-webkit-scrollbar-track { background: #333333; }
 ::-webkit-scrollbar-thumb { background-color: var(--highlight-color); border-radius: var(--radius-lg); border-width: 0; box-shadow: 2px 2px 3px #111111; }
@@ -31,11 +31,16 @@ async function createSplash() {
  const splash = `
    <div id="splash" class="splash" style="background: ${dark ? 'black' : 'white'}">
      <div class="loading"><div class="loader"></div></div>
+      <div id="motd" class="motd""></div>
    </div>`;
  document.body.insertAdjacentHTML('beforeend', splash);
  await preloadImages();
-  const imgElement = `<div class="splash-img" alt="logo" style="background-image: url(file=html/logo-bg-${dark ? 'dark' : 'light'}.jpg), url(file=html/logo-bg-${num}.jpg); background-blend-mode: ${dark ? 'multiply' : 'lighten'}"></div>`;
-  document.getElementById('splash').insertAdjacentHTML('afterbegin', imgElement);
+  const imgEl = `<div id="spash-img" class="splash-img" alt="logo" style="background-image: url(file=html/logo-bg-${dark ? 'dark' : 'light'}.jpg), url(file=html/logo-bg-${num}.jpg); background-blend-mode: ${dark ? 'multiply' : 'lighten'}"></div>`;
+  document.getElementById('splash').insertAdjacentHTML('afterbegin', imgEl);
+  fetch('/sdapi/v1/motd')
+    .then((res) => res.text())
+    .then((text) => document.getElementById('motd').innerHTML = text.replace(/["]+/g, ''))
+    .catch((err) => console.error('getMOTD:', err));
 }

 async function removeSplash() {
@@ -34,11 +34,16 @@ body, button, input, select, textarea { font-family: var(--font);}
 button { font-size: 1.2rem; max-width: 400px; }
 img { background-color: var(--background-color); }
 input[type=checkbox] { background-color: transparent !important; }
-input[type=range] { height: var(--line-sm); appearance: none; margin-top: 0; min-width: 160px; background-color: var(--background-color); width: 100%; background: transparent; }
-input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: var(--input-background-fill); border-radius: var(--radius-lg); border: 0px solid #222222; }
-input[type=range]::-moz-range-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: var(--input-background-fill); border-radius: var(--radius-lg); border: 0px solid #222222; }
-input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: var(--radius-lg); background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; }
-input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: var(--radius-lg); background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; }
+input[type=range] { height: var(--line-sm) !important; appearance: none !important; margin-top: 0 !important; min-width: 160px !important;
+  background-color: var(--background-color) !important; width: 100% !important; background: transparent !important; }
+input[type=range]::-webkit-slider-runnable-track { width: 100% !important; height: var(--line-sm) !important; cursor: pointer !important; box-shadow: 2px 2px 3px #111111 !important;
+  background: var(--input-background-fill) !important; border-radius: var(--radius-lg) !important; border: 0px solid #222222 !important; }
+input[type=range]::-moz-range-track { width: 100% !important; height: var(--line-sm) !important; cursor: pointer !important; box-shadow: 2px 2px 3px #111111 !important; background:
+  var(--input-background-fill) !important; border-radius: var(--radius-lg) !important; border: 0px solid #222222 !important; }
+input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111 !important; border: 0px solid #000000 !important; height: var(--line-sm) !important; width: var(--line-sm) !important;
+  border-radius: var(--radius-lg) !important; background: var(--highlight-color) !important; cursor: pointer !important; appearance: none !important; margin-top: 0px !important; }
+input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111 !important; border: 0px solid #000000 !important; height: var(--line-sm) !important; width: var(--line-sm) !important;
+  border-radius: var(--radius-lg) !important; background: var(--highlight-color) !important; cursor: pointer !important; appearance: none !important; margin-top: 0px !important; }
 ::-webkit-scrollbar { width: 12px; }
 ::-webkit-scrollbar-track { background: #3a343a; }
 ::-webkit-scrollbar-thumb { background-color: var(--highlight-color); border-radius: var(--radius-lg); border-width: 0; box-shadow: 2px 2px 3px #111111; }
@@ -253,6 +253,7 @@ div.controlnet_main_options { display: grid; grid-template-columns: 1fr 1fr; gri

 /* loader */
 .splash { position: fixed; top: 0; left: 0; width: 100vw; height: 100vh; z-index: 1000; display: block; text-align: center; }
+.motd { margin-top: 2em; color: var(--body-text-color-subdued); font-family: monospace; font-variant: all-petite-caps; }
 .splash-img { margin: 10% auto 0 auto; width: 512px; background-repeat: no-repeat; height: 512px; animation: color 10s infinite alternate; }
 .loading { color: white; position: absolute; top: 20%; left: 50%; transform: translateX(-50%); }
 .loader { width: 300px; height: 300px; border: var(--spacing-md) solid transparent; border-radius: 50%; border-top: var(--spacing-md) solid var(--primary-600); animation: spin 4s linear infinite; position: relative; }
@@ -371,7 +371,7 @@ function create_theme_element() {
 }

 function toggleCompact(val) {
-  log('toggleCompact', val);
+  // log('toggleCompact', val);
  if (val) {
    gradioApp().style.setProperty('--layout-gap', 'var(--spacing-md)');
    gradioApp().querySelectorAll('input[type=range]').forEach((el) => el.classList.add('hidden'));
@@ -9,7 +9,7 @@ from fastapi import FastAPI, APIRouter, Depends, Request
 from fastapi.security import HTTPBasic, HTTPBasicCredentials
 from fastapi.exceptions import HTTPException
 from PIL import PngImagePlugin,Image
-
+import requests
 import piexif
 import piexif.helper
 import gradio as gr
@@ -147,6 +147,7 @@ class Api:
        self.add_api_route("/sdapi/v1/script-info", self.get_script_info, methods=["GET"], response_model=List[models.ScriptInfo])
        self.add_api_route("/sdapi/v1/log", self.get_log_buffer, methods=["GET"], response_model=List) # bypass auth
        self.add_api_route("/sdapi/v1/start", self.session_start, methods=["GET"])
+        self.add_api_route("/sdapi/v1/motd", self.get_motd, methods=["GET"], response_model=str)
        self.add_api_route("/sdapi/v1/extra-networks", self.get_extra_networks, methods=["GET"], response_model=List[models.ExtraNetworkItem])
        self.default_script_arg_txt2img = []
        self.default_script_arg_img2img = []
@@ -172,6 +173,19 @@ class Api:
        shared.log.info(f'Browser session: client={req.client.host} agent={agent}')
        return {}

+    def get_motd(self):
+        from installer import get_version
+        motd = ''
+        ver = get_version()
+        if ver.get('updated', None) is not None:
+            motd = f"version <b>{ver['hash']} {ver['updated']}</b> <span style='color: var(--primary-500)'>{ver['url'].split('/')[-1]}</span><br>"
+        if shared.opts.motd:
+            res = requests.get('https://vladmandic.github.io/automatic/motd', timeout=10)
+            if res.status_code == 200:
+                shared.log.info(f'MOTD: {res.text}')
+                motd += res.text
+        return motd
+
    def get_selectable_script(self, script_name, script_runner):
        if script_name is None or script_name == "":
            return None, None
@@ -598,7 +598,6 @@ def save_image(image, path, basename, seed=None, prompt=None, extension='jpg', i
        filename = filename[:max_name_len - max(4, len(extension))]
        params.filename = filename + extension
    txt_fullfn = f"{filename}.txt" if shared.opts.save_txt and len(exifinfo) > 0 else None
-
    save_queue.put((params.image, filename, extension, params, exifinfo, txt_fullfn)) # actual save is executed in a thread that polls data from queue
    save_queue.join()

@@ -26,7 +26,7 @@ def setup_middleware(app: FastAPI, cmd_opts):
    from fastapi.middleware.gzip import GZipMiddleware
    app.user_middleware = [x for x in app.user_middleware if x.cls.__name__ != 'CORSMiddleware']
    app.middleware_stack = None # reset current middleware to allow modifying user provided list
-    app.add_middleware(GZipMiddleware, minimum_size=1024)
+    app.add_middleware(GZipMiddleware, minimum_size=2048)
    if cmd_opts.cors_origins and cmd_opts.cors_regex:
        app.add_middleware(CORSMiddleware, allow_origins=cmd_opts.cors_origins.split(','), allow_origin_regex=cmd_opts.cors_regex, allow_methods=['*'], allow_credentials=True, allow_headers=['*'])
    elif cmd_opts.cors_origins:
@@ -22,6 +22,7 @@ import modules.lowvram
 import modules.masking
 import modules.paths
 import modules.scripts
+import modules.script_callbacks
 import modules.prompt_parser
 import modules.extra_networks
 import modules.face_restoration
@@ -167,6 +168,7 @@ class StableDiffusionProcessing:
        self.s_tmin = shared.opts.s_tmin
        self.s_tmax = float('inf')  # not representable as a standard ui option
        self.comments = {}
+        self.is_api = False
        shared.opts.data['clip_skip'] = clip_skip

    @property
@@ -364,8 +366,7 @@ class Processed:
        return self.token_merging_ratio_hr if for_hr else self.token_merging_ratio


-# from https://discuss.pytorch.org/t/help-regarding-slerp-function-for-generative-model-sampling/32475/3
-def slerp(val, low, high):
+def slerp(val, low, high): # from https://discuss.pytorch.org/t/help-regarding-slerp-function-for-generative-model-sampling/32475/3
    low_norm = low/torch.norm(low, dim=1, keepdim=True)
    high_norm = high/torch.norm(high, dim=1, keepdim=True)
    dot = (low_norm*high_norm).sum(1)
@@ -382,7 +383,6 @@ def slerp(val, low, high):
 def create_random_tensors(shape, seeds, subseeds=None, subseed_strength=0.0, seed_resize_from_h=0, seed_resize_from_w=0, p=None):
    eta_noise_seed_delta = shared.opts.eta_noise_seed_delta or 0
    xs = []
-
    # if we have multiple seeds, this means we are working with batch size>1; this then
    # enables the generation of additional tensors with noise that the sampler will use during its processing.
    # Using those pre-generated tensors instead of simple torch.randn allows a batch with seeds [100, 101] to
@@ -391,24 +391,19 @@ def create_random_tensors(shape, seeds, subseeds=None, subseed_strength=0.0, see
        sampler_noises = [[] for _ in range(p.sampler.number_of_needed_noises(p))]
    else:
        sampler_noises = None
-
    for i, seed in enumerate(seeds):
        noise_shape = shape if seed_resize_from_h <= 0 or seed_resize_from_w <= 0 else (shape[0], seed_resize_from_h//8, seed_resize_from_w//8)
-
        subnoise = None
        if subseeds is not None:
            subseed = 0 if i >= len(subseeds) else subseeds[i]
            subnoise = devices.randn(subseed, noise_shape)
-
        # randn results depend on device; gpu and cpu get different results for same seed;
        # the way I see it, it's better to do this on CPU, so that everyone gets same result;
        # but the original script had it like this, so I do not dare change it for now because
        # it will break everyone's seeds.
        noise = devices.randn(seed, noise_shape)
-
        if subnoise is not None:
            noise = slerp(subseed_strength, noise, subnoise)
-
        if noise_shape != shape:
            x = devices.randn(seed, shape)
            dx = (shape[2] - noise_shape[2]) // 2
@@ -421,19 +416,15 @@ def create_random_tensors(shape, seeds, subseeds=None, subseed_strength=0.0, see
            dy = max(-dy, 0)
            x[:, ty:ty+h, tx:tx+w] = noise[:, dy:dy+h, dx:dx+w]
            noise = x
-
        if sampler_noises is not None:
            cnt = p.sampler.number_of_needed_noises(p)
            if eta_noise_seed_delta > 0:
                torch.manual_seed(seed + eta_noise_seed_delta)
            for j in range(cnt):
                sampler_noises[j].append(devices.randn_without_seed(tuple(noise_shape)))
-
        xs.append(noise)
-
    if sampler_noises is not None:
        p.sampler.sampler_noises = [torch.stack(n).to(shared.device) for n in sampler_noises]
-
    x = torch.stack(xs).to(shared.device)
    return x

@@ -532,10 +523,9 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_su
        args["Face restoration"] = shared.opts.face_restoration_model
    if 'color' in p.ops:
        args["Color correction"] = True
-
+    # embeddings
    if hasattr(modules.sd_hijack.model_hijack, 'embedding_db') and len(modules.sd_hijack.model_hijack.embedding_db.embeddings_used) > 0: # this is for original hijaacked models only, diffusers are handled separately
        args["Embeddings"] = ', '.join(modules.sd_hijack.model_hijack.embedding_db.embeddings_used)
-
    # samplers
    args["Sampler ENSD"] = shared.opts.eta_noise_seed_delta if shared.opts.eta_noise_seed_delta != 0 and modules.sd_samplers_common.is_sampler_using_eta_noise_seed_delta(p) else None
    args["Sampler ENSM"] = p.initial_noise_multiplier if getattr(p, 'initial_noise_multiplier', 1.0) != 1.0 else None
@@ -645,6 +635,8 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
            modules.sd_models.apply_token_merging(p.sd_model, p.get_token_merging_ratio())
            modules.sd_hijack_freeu.apply_freeu(p, shared.backend == shared.Backend.ORIGINAL)

+        modules.script_callbacks.before_process_callback(p)
+
        if shared.cmd_opts.profile:
            """
            import torch.profiler # pylint: disable=redefined-outer-name
@@ -656,7 +648,8 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
            import cProfile
            pr = cProfile.Profile()
            pr.enable()
-            res = process_images_inner(p)
+            with context_hypertile_vae(p), context_hypertile_unet(p):
+                res = process_images_inner(p)
            print_profile(pr, 'Torch')
        else:
            with context_hypertile_vae(p), context_hypertile_unet(p):
@@ -664,6 +657,7 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
    finally:
        if not shared.opts.cuda_compile:
            modules.sd_models.apply_token_merging(p.sd_model, 0)
+        modules.script_callbacks.after_process_callback(p)
        if p.override_settings_restore_afterwards: # restore opts to original state
            for k, v in stored_opts.items():
                setattr(shared.opts, k, v)
@@ -909,7 +903,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                if shared.opts.grid_save:
                    images.save_image(grid, p.outpath_grids, "", p.all_seeds[0], p.all_prompts[0], shared.opts.grid_format, info=infotext(-1), short_filename=not shared.opts.grid_extended_filename, p=p, grid=True, suffix="-grid") # main save grid

-    if not p.disable_extra_networks and extra_network_data:
+    if not p.disable_extra_networks:
        modules.extra_networks.deactivate(p, extra_network_data)

    res = Processed(
@@ -1036,6 +1030,8 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
        self.ops.append('txt2img')
        hypertile_set(self)
        self.sampler = modules.sd_samplers.create_sampler(self.sampler_name, self.sd_model)
+        if hasattr(self.sampler, "initialize"):
+            self.sampler.initialize(self)
        x = create_random_tensors([4, self.height // 8, self.width // 8], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self)
        samples = self.sampler.sample(self, x, conditioning, unconditional_conditioning, image_conditioning=self.txt2img_image_conditioning(x))
        if not self.enable_hr or shared.state.interrupted or shared.state.skipped:
@@ -1084,9 +1080,12 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
                    self.ops.append('hires')
                    devices.torch_gc() # GC now before running the next img2img to prevent running out of memory
                    self.sampler = modules.sd_samplers.create_sampler(self.latent_sampler or self.sampler_name, self.sd_model)
+                    if hasattr(self.sampler, "initialize"):
+                        self.sampler.initialize(self)
                    samples = samples[:, :, self.truncate_y//2:samples.shape[2]-(self.truncate_y+1)//2, self.truncate_x//2:samples.shape[3]-(self.truncate_x+1)//2]
                    noise = create_random_tensors(samples.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=subseed_strength, p=self)
                    modules.sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio(for_hr=True))
+                    hypertile_set(self, hr=True)
                    samples = self.sampler.sample_img2img(self, samples, noise, conditioning, unconditional_conditioning, steps=self.hr_second_pass_steps or self.steps, image_conditioning=image_conditioning)
                    modules.sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio())
                else:
@@ -1138,6 +1137,8 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
        if self.sampler_name == "PLMS":
            self.sampler_name = 'UniPC'
        self.sampler = modules.sd_samplers.create_sampler(self.sampler_name, self.sd_model)
+        if hasattr(self.sampler, "initialize"):
+            self.sampler.initialize(self)

        if self.image_mask is not None:
            self.ops.append('inpaint')
@@ -14,6 +14,7 @@ import modules.taesd.sd_vae_taesd as sd_vae_taesd
 import modules.images as images
 from modules.processing import StableDiffusionProcessing
 import modules.prompt_parser_diffusers as prompt_parser_diffusers
+from modules.sd_hijack_hypertile import hypertile_set


 def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_prompts):
@@ -256,6 +257,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                pass
                # shared.log.debug(f'Diffuser not supported: pipeline={pipeline.__class__.__name__} task={sd_models.get_diffusers_task(model)} arg={arg}')
        # shared.log.debug(f'Diffuser pipeline: {pipeline.__class__.__name__} possible={possible}')
+        hypertile_set(p, hr=hasattr(p, 'init_images') and len(p.init_images) > 0)
        clean = args.copy()
        clean.pop('callback', None)
        clean.pop('callback_steps', None)
@@ -321,7 +323,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro

    p.extra_generation_params['Pipeline'] = shared.sd_model.__class__.__name__

-    cross_attention_kwargs={}
    if len(getattr(p, 'init_images', [])) > 0:
        while len(p.init_images) < len(prompts):
            p.init_images.append(p.init_images[-1])
@@ -92,6 +92,8 @@ class ImageGridLoopParams:
 ScriptCallback = namedtuple("ScriptCallback", ["script", "callback"])
 callback_map = dict(
    callbacks_app_started=[],
+    callbacks_before_process=[],
+    callbacks_after_process=[],
    callbacks_model_loaded=[],
    callbacks_ui_tabs=[],
    callbacks_ui_train_tabs=[],
@@ -134,6 +136,26 @@ def app_started_callback(demo: Optional[Blocks], app: FastAPI):
            report_exception(e, c, 'app_started_callback')


+def before_process_callback(p):
+    for c in callback_map['callbacks_before_process']:
+        try:
+            t0 = time.time()
+            c.callback(p)
+            timer(t0, c.script, 'before_process')
+        except Exception as e:
+            report_exception(e, c, 'before_process_callback')
+
+
+def after_process_callback(p):
+    for c in callback_map['callbacks_after_process']:
+        try:
+            t0 = time.time()
+            c.callback(p)
+            timer(t0, c.script, 'after_process')
+        except Exception as e:
+            report_exception(e, c, 'after_process_callback')
+
+
 def app_reload_callback():
    for c in callback_map['callbacks_on_reload']:
        try:
@@ -334,6 +356,16 @@ def on_app_started(callback):
    add_callback(callback_map['callbacks_app_started'], callback)


+def on_before_process(callback):
+    """register a function to be called just before processing starts"""
+    add_callback(callback_map['callbacks_before_process'], callback)
+
+
+def on_after_process(callback):
+    """register a function to be called just after processing ends"""
+    add_callback(callback_map['callbacks_after_process'], callback)
+
+
 def on_before_reload(callback):
    """register a function to be called just before the server reloads."""
    add_callback(callback_map['callbacks_on_reload'], callback)
@@ -90,7 +90,7 @@ def split_attention(layer: nn.Module, tile_size: int=256, min_tile_size: int=256
                        width = 8 * w
                        max_w = max(max_w, w)
                        reset_nws()
-                down_ratio = height // 8 // h
+                down_ratio = max(height // 8 // h, 1)
                curr_depth = round(math.log(down_ratio, 2))
                # scale-up the tile-size the deeper we go
                nh = max(1, nh // down_ratio)
@@ -136,6 +136,9 @@ def context_hypertile_vae(p):
    from modules import shared
    if p.sd_model is None or not shared.opts.hypertile_vae_enabled:
        return nullcontext()
+    if shared.opts.cross_attention_optimization == 'Sub-quadratic':
+        shared.log.warning('Hypertile UNet is not compatible with Sub-quadratic cross-attention optimization')
+        return nullcontext()
    vae = getattr(p.sd_model, "vae", None) if shared.backend == shared.Backend.DIFFUSERS else getattr(p.sd_model, "first_stage_model", None)
    if vae is None:
        shared.log.warning('Hypertile VAE is enabled but no VAE model was found')
@@ -156,6 +159,9 @@ def context_hypertile_unet(p):
    from modules import shared
    if p.sd_model is None or not shared.opts.hypertile_unet_enabled:
        return nullcontext()
+    if shared.opts.cross_attention_optimization == 'Sub-quadratic' and not shared.cmd_opts.experimental:
+        shared.log.warning('Hypertile UNet is not compatible with Sub-quadratic cross-attention optimization')
+        return nullcontext()
    unet = getattr(p.sd_model, "unet", None) if shared.backend == shared.Backend.DIFFUSERS else getattr(p.sd_model.model, "diffusion_model", None)
    if unet is None:
        shared.log.warning('Hypertile Unet is enabled but no Unet model was found')
@@ -166,9 +172,12 @@ def context_hypertile_unet(p):
        return split_attention(unet, tile_size=shared.opts.hypertile_unet_tile, min_tile_size=128, swap_size=1)


-def hypertile_set(p):
+def hypertile_set(p, hr=False):
+    from modules import shared
    global height, width, error_reported, reset_needed # pylint: disable=global-statement
+    if not shared.opts.hypertile_unet_enabled:
+        return
    error_reported = False
-    height=p.height
-    width=p.width
+    height=p.height if not hr else getattr(p, 'hr_upscale_to_y', p.height)
+    width=p.width if not hr else getattr(p, 'hr_upscale_to_x', p.width)
    reset_needed = True
@@ -1240,7 +1240,7 @@ def apply_token_merging(sd_model, token_merging_ratio=0):
    except Exception:
        pass
    if token_merging_ratio > 0:
-        if shared.opts.hypertile_unet_enabled:
+        if shared.opts.hypertile_unet_enabled and not shared.cmd_opts.experimental:
            shared.log.warning('Token merging not supported with HyperTile for UNet')
            return
        try:
@@ -0,0 +1,231 @@
+# TODO: implementation missing
+
+import torch
+from modules import prompt_parser, devices, sd_samplers_common
+
+from modules.shared import opts, state
+import modules.shared as shared
+from modules.script_callbacks import CFGDenoiserParams, cfg_denoiser_callback
+from modules.script_callbacks import CFGDenoisedParams, cfg_denoised_callback
+from modules.script_callbacks import AfterCFGCallbackParams, cfg_after_cfg_callback
+
+
+def catenate_conds(conds):
+    if not isinstance(conds[0], dict):
+        return torch.cat(conds)
+
+    return {key: torch.cat([x[key] for x in conds]) for key in conds[0].keys()}
+
+
+def subscript_cond(cond, a, b):
+    if not isinstance(cond, dict):
+        return cond[a:b]
+
+    return {key: vec[a:b] for key, vec in cond.items()}
+
+
+def pad_cond(tensor, repeats, empty):
+    if not isinstance(tensor, dict):
+        return torch.cat([tensor, empty.repeat((tensor.shape[0], repeats, 1))], axis=1)
+
+    tensor['crossattn'] = pad_cond(tensor['crossattn'], repeats, empty)
+    return tensor
+
+
+class CFGDenoiser(torch.nn.Module):
+    """
+    Classifier free guidance denoiser. A wrapper for stable diffusion model (specifically for unet)
+    that can take a noisy picture and produce a noise-free picture using two guidances (prompts)
+    instead of one. Originally, the second prompt is just an empty string, but we use non-empty
+    negative prompt.
+    """
+
+    def __init__(self, sampler):
+        super().__init__()
+        self.model_wrap = None
+        self.mask = None
+        self.nmask = None
+        self.init_latent = None
+        self.steps = None
+        """number of steps as specified by user in UI"""
+
+        self.total_steps = None
+        """expected number of calls to denoiser calculated from self.steps and specifics of the selected sampler"""
+
+        self.step = 0
+        self.image_cfg_scale = None
+        self.padded_cond_uncond = False
+        self.sampler = sampler
+        self.model_wrap = None
+        self.p = None
+        self.mask_before_denoising = False
+
+    @property
+    def inner_model(self):
+        raise NotImplementedError()
+
+    def combine_denoised(self, x_out, conds_list, uncond, cond_scale):
+        denoised_uncond = x_out[-uncond.shape[0]:]
+        denoised = torch.clone(denoised_uncond)
+
+        for i, conds in enumerate(conds_list):
+            for cond_index, weight in conds:
+                denoised[i] += (x_out[cond_index] - denoised_uncond[i]) * (weight * cond_scale)
+
+        return denoised
+
+    def combine_denoised_for_edit_model(self, x_out, cond_scale):
+        out_cond, out_img_cond, out_uncond = x_out.chunk(3)
+        denoised = out_uncond + cond_scale * (out_cond - out_img_cond) + self.image_cfg_scale * (out_img_cond - out_uncond)
+
+        return denoised
+
+    def get_pred_x0(self, x_in, x_out, sigma): # pylint: disable=unused-argument
+        return x_out
+
+    def update_inner_model(self):
+        self.model_wrap = None
+
+        c, uc = self.p.get_conds()
+        self.sampler.sampler_extra_args['cond'] = c
+        self.sampler.sampler_extra_args['uncond'] = uc
+
+    def forward(self, x, sigma, uncond, cond, cond_scale, s_min_uncond, image_cond):
+        if state.interrupted or state.skipped:
+            raise sd_samplers_common.InterruptedException
+
+        # if sd_samplers_common.apply_refiner(self): # TODO implementation missing
+        #     cond = self.sampler.sampler_extra_args['cond']
+        #    uncond = self.sampler.sampler_extra_args['uncond']
+
+        # at self.image_cfg_scale == 1.0 produced results for edit model are the same as with normal sampling,
+        # so is_edit_model is set to False to support AND composition.
+        is_edit_model = shared.sd_model.cond_stage_key == "edit" and self.image_cfg_scale is not None and self.image_cfg_scale != 1.0
+
+        conds_list, tensor = prompt_parser.reconstruct_multicond_batch(cond, self.step)
+        uncond = prompt_parser.reconstruct_cond_batch(uncond, self.step)
+
+        assert not is_edit_model or all(len(conds) == 1 for conds in conds_list), "AND is not supported for InstructPix2Pix checkpoint (unless using Image CFG scale = 1.0)"
+
+        if self.mask_before_denoising and self.mask is not None:
+            x = self.init_latent * self.mask + self.nmask * x
+
+        batch_size = len(conds_list)
+        repeats = [len(conds_list[i]) for i in range(batch_size)]
+
+        if shared.sd_model.model.conditioning_key == "crossattn-adm":
+            image_uncond = torch.zeros_like(image_cond)
+            make_condition_dict = lambda c_crossattn, c_adm: {"c_crossattn": [c_crossattn], "c_adm": c_adm} # pylint: disable=unnecessary-lambda-assignment
+        else:
+            image_uncond = image_cond
+            if isinstance(uncond, dict):
+                make_condition_dict = lambda c_crossattn, c_concat: {**c_crossattn, "c_concat": [c_concat]} # pylint: disable=unnecessary-lambda-assignment
+            else:
+                make_condition_dict = lambda c_crossattn, c_concat: {"c_crossattn": [c_crossattn], "c_concat": [c_concat]} # pylint: disable=unnecessary-lambda-assignment
+
+        if not is_edit_model:
+            x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x])
+            sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma])
+            image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_uncond])
+        else:
+            x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x] + [x])
+            sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma] + [sigma])
+            image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_uncond] + [torch.zeros_like(self.init_latent)])
+
+        denoiser_params = CFGDenoiserParams(x_in, image_cond_in, sigma_in, state.sampling_step, state.sampling_steps, tensor, uncond)
+        cfg_denoiser_callback(denoiser_params)
+        x_in = denoiser_params.x
+        image_cond_in = denoiser_params.image_cond
+        sigma_in = denoiser_params.sigma
+        tensor = denoiser_params.text_cond
+        uncond = denoiser_params.text_uncond
+        skip_uncond = False
+
+        # alternating uncond allows for higher thresholds without the quality loss normally expected from raising it
+        if self.step % 2 and s_min_uncond > 0 and sigma[0] < s_min_uncond and not is_edit_model:
+            skip_uncond = True
+            x_in = x_in[:-batch_size]
+            sigma_in = sigma_in[:-batch_size]
+
+        self.padded_cond_uncond = False
+        if shared.opts.pad_cond_uncond and tensor.shape[1] != uncond.shape[1]:
+            empty = shared.sd_model.cond_stage_model_empty_prompt
+            num_repeats = (tensor.shape[1] - uncond.shape[1]) // empty.shape[1]
+
+            if num_repeats < 0:
+                tensor = pad_cond(tensor, -num_repeats, empty)
+                self.padded_cond_uncond = True
+            elif num_repeats > 0:
+                uncond = pad_cond(uncond, num_repeats, empty)
+                self.padded_cond_uncond = True
+
+        if tensor.shape[1] == uncond.shape[1] or skip_uncond:
+            if is_edit_model:
+                cond_in = catenate_conds([tensor, uncond, uncond])
+            elif skip_uncond:
+                cond_in = tensor
+            else:
+                cond_in = catenate_conds([tensor, uncond])
+
+            if shared.opts.batch_cond_uncond:
+                x_out = self.inner_model(x_in, sigma_in, cond=make_condition_dict(cond_in, image_cond_in))
+            else:
+                x_out = torch.zeros_like(x_in)
+                for batch_offset in range(0, x_out.shape[0], batch_size):
+                    a = batch_offset
+                    b = a + batch_size
+                    x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond=make_condition_dict(subscript_cond(cond_in, a, b), image_cond_in[a:b]))
+        else:
+            x_out = torch.zeros_like(x_in)
+            batch_size = batch_size*2 if shared.opts.batch_cond_uncond else batch_size
+            for batch_offset in range(0, tensor.shape[0], batch_size):
+                a = batch_offset
+                b = min(a + batch_size, tensor.shape[0])
+
+                if not is_edit_model:
+                    c_crossattn = subscript_cond(tensor, a, b)
+                else:
+                    c_crossattn = torch.cat([tensor[a:b]], uncond)
+
+                x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond=make_condition_dict(c_crossattn, image_cond_in[a:b]))
+
+            if not skip_uncond:
+                x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond=make_condition_dict(uncond, image_cond_in[-uncond.shape[0]:]))
+
+        denoised_image_indexes = [x[0][0] for x in conds_list]
+        if skip_uncond:
+            fake_uncond = torch.cat([x_out[i:i+1] for i in denoised_image_indexes])
+            x_out = torch.cat([x_out, fake_uncond])  # we skipped uncond denoising, so we put cond-denoised image to where the uncond-denoised image should be
+
+        denoised_params = CFGDenoisedParams(x_out, state.sampling_step, state.sampling_steps, self.inner_model)
+        cfg_denoised_callback(denoised_params)
+
+        devices.test_for_nans(x_out, "unet")
+
+        if is_edit_model:
+            denoised = self.combine_denoised_for_edit_model(x_out, cond_scale)
+        elif skip_uncond:
+            denoised = self.combine_denoised(x_out, conds_list, uncond, 1.0)
+        else:
+            denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale)
+
+        if not self.mask_before_denoising and self.mask is not None:
+            denoised = self.init_latent * self.mask + self.nmask * denoised
+
+        self.sampler.last_latent = self.get_pred_x0(torch.cat([x_in[i:i + 1] for i in denoised_image_indexes]), torch.cat([x_out[i:i + 1] for i in denoised_image_indexes]), sigma)
+
+        if opts.live_preview_content == "Prompt":
+            preview = self.sampler.last_latent
+        elif opts.live_preview_content == "Negative prompt":
+            preview = self.get_pred_x0(x_in[-uncond.shape[0]:], x_out[-uncond.shape[0]:], sigma)
+        else:
+            preview = self.get_pred_x0(torch.cat([x_in[i:i+1] for i in denoised_image_indexes]), torch.cat([denoised[i:i+1] for i in denoised_image_indexes]), sigma)
+
+        sd_samplers_common.store_latent(preview)
+
+        after_cfg_callback_params = AfterCFGCallbackParams(denoised, state.sampling_step, state.sampling_steps)
+        cfg_after_cfg_callback(after_cfg_callback_params)
+        denoised = after_cfg_callback_params.x
+
+        self.step += 1
+        return denoised
@@ -128,26 +128,24 @@ class VanillaStableDiffusionSampler:
        self.update_step(x)

    def initialize(self, p):
-        if self.is_ddim:
-            self.eta = p.eta if p.eta is not None else shared.opts.scheduler_eta
-        else:
-            self.eta = 0.0
-        if self.eta != 0.0:
-            p.extra_generation_params["Sampler Eta"] = self.eta
-
-        if self.is_unipc:
-            keys = [
-                ('Solver order', 'schedulers_solver_order'),
-                ('Sampler low order', 'schedulers_use_loworder'),
-                ('UniPC variant', 'uni_pc_variant'),
-                ('UniPC skip type', 'uni_pc_skip_type'),
-            ]
-
-            for name, key in keys:
-                v = getattr(shared.opts, key)
-                if v != shared.opts.get_default(key):
-                    p.extra_generation_params[name] = v
-
+        if p is not None:
+            if self.is_ddim:
+                self.eta = p.eta if p.eta is not None else shared.opts.scheduler_eta
+            else:
+                self.eta = 0.0
+            if self.eta != 0.0:
+                p.extra_generation_params["Sampler Eta"] = self.eta
+            if self.is_unipc:
+                keys = [
+                    ('Solver order', 'schedulers_solver_order'),
+                    ('Sampler low order', 'schedulers_use_loworder'),
+                    ('UniPC variant', 'uni_pc_variant'),
+                    ('UniPC skip type', 'uni_pc_skip_type'),
+                ]
+                for name, key in keys:
+                    v = getattr(shared.opts, key)
+                    if v != shared.opts.get_default(key):
+                        p.extra_generation_params[name] = v
        for fieldname in ['p_sample_ddim', 'p_sample_plms']:
            if hasattr(self.sampler, fieldname):
                setattr(self.sampler, fieldname, self.p_sample_ddim_hook)
@@ -277,10 +277,8 @@ class KDiffusionSampler:
            self.config.options['scheduler'] = shared.opts.data.get('schedulers_sigma', None)
        if p is None:
            return
-
        self.model_wrap_cfg.mask = p.mask if hasattr(p, 'mask') else None
        self.model_wrap_cfg.nmask = p.nmask if hasattr(p, 'nmask') else None
-        self.model_wrap_cfg.step = 0
        self.model_wrap_cfg.image_cfg_scale = getattr(p, 'image_cfg_scale', None)
        self.eta = p.eta if p.eta is not None else shared.opts.scheduler_eta
        self.s_min_uncond = getattr(p, 's_min_uncond', 0.0)
@@ -386,7 +386,7 @@ else: # cuda


 options_templates.update(options_section(('sd', "Execution & Models"), {
-    "sd_backend": OptionInfo("diffusers" if cmd_opts.use_openvino else "original", "Execution backend", gr.Radio, lambda: {"choices": ["original", "diffusers"] }),
+    "sd_backend": OptionInfo("diffusers" if cmd_opts.use_openvino else "original", "Execution backend", gr.Radio, {"choices": ["original", "diffusers"] }),
    "sd_checkpoint_autoload": OptionInfo(True, "Model autoload on server start"),
    "sd_model_checkpoint": OptionInfo(default_checkpoint, "Base model", gr.Dropdown, lambda: {"choices": list_checkpoint_tiles()}, refresh=refresh_checkpoints),
    "sd_model_refiner": OptionInfo('None', "Refiner model", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints),
@@ -394,7 +394,7 @@ options_templates.update(options_section(('sd', "Execution & Models"), {
    "sd_model_dict": OptionInfo('None', "Use baseline data from a different model", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints),
    "stream_load": OptionInfo(False, "Load models using stream loading method"),
    "model_reuse_dict": OptionInfo(False, "When loading models attempt to reuse previous model dictionary", gr.Checkbox, {"visible": False}),
-    "prompt_attention": OptionInfo("Full parser", "Prompt attention parser", gr.Radio, lambda: {"choices": ["Full parser", "Compel parser", "A1111 parser", "Fixed attention"] }),
+    "prompt_attention": OptionInfo("Full parser", "Prompt attention parser", gr.Radio, {"choices": ["Full parser", "Compel parser", "A1111 parser", "Fixed attention"] }),
    "prompt_mean_norm": OptionInfo(True, "Prompt attention mean normalization"),
    "comma_padding_backtrack": OptionInfo(20, "Prompt padding for long prompts", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }),
    "sd_checkpoint_cache": OptionInfo(0, "Number of cached models", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}),
@@ -404,8 +404,8 @@ options_templates.update(options_section(('sd', "Execution & Models"), {

 options_templates.update(options_section(('cuda', "Compute Settings"), {
    "math_sep": OptionInfo("<h2>Execution precision</h2>", "", gr.HTML),
-    "precision": OptionInfo("Autocast", "Precision type", gr.Radio, lambda: {"choices": ["Autocast", "Full"]}),
-    "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" or cmd_opts.use_openvino else "BF16" if devices.backend == "ipex" else "FP16", "Device precision type", gr.Radio, lambda: {"choices": ["FP32", "FP16", "BF16"]}),
+    "precision": OptionInfo("Autocast", "Precision type", gr.Radio, {"choices": ["Autocast", "Full"]}),
+    "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" or cmd_opts.use_openvino else "BF16" if devices.backend == "ipex" else "FP16", "Device precision type", gr.Radio, {"choices": ["FP32", "FP16", "BF16"]}),
    "no_half": OptionInfo(False, "Use full precision for model (--no-half)", None, None, None),
    "no_half_vae": OptionInfo(False, "Use full precision for VAE (--no-half-vae)"),
    "upcast_sampling": OptionInfo(True if sys.platform == "darwin" else False, "Enable upcast sampling"),
@@ -416,7 +416,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), {

    "cross_attention_sep": OptionInfo("<h2>Cross-attention</h2>", "", gr.HTML),
    "cross_attention_optimization": OptionInfo(cross_attention_optimization_default, "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }),
-    "cross_attention_options": OptionInfo([], "Cross-attention advanced options", gr.CheckboxGroup, lambda: {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention']}),
+    "cross_attention_options": OptionInfo([], "Cross-attention advanced options", gr.CheckboxGroup, {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention']}),
    "sub_quad_sep": OptionInfo("<h3>Sub-quadratic options</h3>", "", gr.HTML),
    "sub_quad_q_chunk_size": OptionInfo(512, "cross-attention query chunk size", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8}),
    "sub_quad_kv_chunk_size": OptionInfo(512, "cross-attention kv chunk size", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8}),
@@ -429,8 +429,8 @@ options_templates.update(options_section(('cuda', "Compute Settings"), {

    "cuda_compile_sep": OptionInfo("<h2>Model Compile</h2>", "", gr.HTML),
    "cuda_compile": OptionInfo(True if cmd_opts.use_openvino else False, "Enable model compile"),
-    "cuda_compile_backend": OptionInfo("openvino_fx" if cmd_opts.use_openvino else "none", "Model compile backend", gr.Radio, lambda: {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex', 'openvino_fx']}),
-    "cuda_compile_mode": OptionInfo("default", "Model compile mode", gr.Radio, lambda: {"choices": ['default', 'reduce-overhead', 'max-autotune']}),
+    "cuda_compile_backend": OptionInfo("openvino_fx" if cmd_opts.use_openvino else "none", "Model compile backend", gr.Radio, {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex', 'openvino_fx']}),
+    "cuda_compile_mode": OptionInfo("default", "Model compile mode", gr.Radio, {"choices": ['default', 'reduce-overhead', 'max-autotune']}),
    "cuda_compile_fullgraph": OptionInfo(False, "Model compile fullgraph"),
    "cuda_compile_precompile": OptionInfo(False, "Model compile precompile"),
    "cuda_compile_verbose": OptionInfo(False, "Model compile verbose mode"),
@@ -438,7 +438,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), {

    "ipex_sep": OptionInfo("<h2>IPEX, DirectML and OpenVINO</h2>", "", gr.HTML),
    "ipex_optimize": OptionInfo(True if devices.backend == "ipex" else False, "Enable IPEX Optimize for Intel GPUs"),
-    "directml_memory_provider": OptionInfo(default_memory_provider, 'DirectML memory stats provider', gr.Radio, lambda: {"choices": memory_providers}),
+    "directml_memory_provider": OptionInfo(default_memory_provider, 'DirectML memory stats provider', gr.Radio, {"choices": memory_providers}),
    "openvino_disable_model_caching": OptionInfo(False, "OpenVINO disable model caching"),
    "openvino_multi_gpu": OptionInfo(False, "OpenVINO use Multi GPU"),
    "openvino_remove_igpu_from_multi": OptionInfo(False, "OpenVINO remove iGPU from Multi GPU"),
@@ -465,7 +465,7 @@ options_templates.update(options_section(('advanced', "Inference Settings"), {

    "inference_other_sep": OptionInfo("<h2>Other</h2>", "", gr.HTML),
    "batch_frame_mode": OptionInfo(False, "Process multiple images in batch in parallel"),
-    "inference_mode": OptionInfo("no-grad", "Torch inference mode", gr.Radio, lambda: {"choices": ["no-grad", "inference-mode", "none"]}),
+    "inference_mode": OptionInfo("no-grad", "Torch inference mode", gr.Radio, {"choices": ["no-grad", "inference-mode", "none"]}),
    "sd_vae_sliced_encode": OptionInfo(False, "VAE Slicing (original)"),
 }))

@@ -475,20 +475,20 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), {
    "diffusers_move_unet": OptionInfo(True, "Move base model to CPU when using VAE"),
    "diffusers_move_refiner": OptionInfo(True, "Move refiner model to CPU when not in use"),
    "diffusers_extract_ema": OptionInfo(True, "Use model EMA weights when possible"),
-    "diffusers_generator_device": OptionInfo("default", "Generator device", gr.Radio, lambda: {"choices": ["default", "cpu"]}),
+    "diffusers_generator_device": OptionInfo("default", "Generator device", gr.Radio, {"choices": ["default", "cpu"]}),
    "diffusers_model_cpu_offload": OptionInfo(False, "Enable model CPU offload (--medvram)"),
    "diffusers_seq_cpu_offload": OptionInfo(False, "Enable sequential CPU offload (--lowvram)"),
-    "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, lambda: {"choices": ['default', 'true', 'false']}),
+    "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, {"choices": ['default', 'true', 'false']}),
    "diffusers_vae_slicing": OptionInfo(True, "Enable VAE slicing"),
    "diffusers_vae_tiling": OptionInfo(False if cmd_opts.use_openvino else True, "Enable VAE tiling"),
    "diffusers_attention_slicing": OptionInfo(False, "Enable attention slicing"),
-    "diffusers_model_load_variant": OptionInfo("default", "Diffusers model loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}),
-    "diffusers_vae_load_variant": OptionInfo("default", "Diffusers VAE loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}),
-    "diffusers_lora_loader": OptionInfo("diffusers" if cmd_opts.use_openvino else "sequential apply", "Diffusers LoRA loading variant", gr.Radio, lambda: {"choices": ['diffusers', 'sequential apply', 'merge and apply']}),
+    "diffusers_model_load_variant": OptionInfo("default", "Diffusers model loading variant", gr.Radio, {"choices": ['default', 'fp32', 'fp16']}),
+    "diffusers_vae_load_variant": OptionInfo("default", "Diffusers VAE loading variant", gr.Radio, {"choices": ['default', 'fp32', 'fp16']}),
+    "diffusers_lora_loader": OptionInfo("diffusers" if cmd_opts.use_openvino else "sequential apply", "Diffusers LoRA loading variant", gr.Radio, {"choices": ['diffusers', 'sequential apply', 'merge and apply']}),
    "diffusers_force_zeros": OptionInfo(True, "Force zeros for prompts when empty"),
    "diffusers_aesthetics_score": OptionInfo(False, "Require aesthetics score"),
    "diffusers_force_inpaint": OptionInfo(False, 'Diffusers force inpaint pipeline'),
-    "diffusers_pooled": OptionInfo("default", "Diffusers SDXL pooled embeds (experimental)", gr.Radio, lambda: {"choices": ['default', 'weighted']}),
+    "diffusers_pooled": OptionInfo("default", "Diffusers SDXL pooled embeds (experimental)", gr.Radio, {"choices": ['default', 'weighted']}),
 }))

 options_templates.update(options_section(('system-paths', "System Paths"), {
@@ -497,7 +497,7 @@ options_templates.update(options_section(('system-paths', "System Paths"), {
    "ckpt_dir": OptionInfo(os.path.join(paths.models_path, 'Stable-diffusion'), "Folder with stable diffusion models", folder=True),
    "diffusers_dir": OptionInfo(os.path.join(paths.models_path, 'Diffusers'), "Folder with Hugggingface models", folder=True),
    "vae_dir": OptionInfo(os.path.join(paths.models_path, 'VAE'), "Folder with VAE files", folder=True),
-    "sd_lora": OptionInfo("", "Add LoRA to prompt", gr.Textbox, {"visible": False}),
+    # "sd_lora": OptionInfo("", "Add LoRA to prompt", gr.Textbox, {"visible": False}),
    "lora_dir": OptionInfo(os.path.join(paths.models_path, 'Lora'), "Folder with LoRA network(s)", folder=True),
    "lyco_dir": OptionInfo(os.path.join(paths.models_path, 'LyCORIS'), "Folder with LyCORIS network(s)", folder=True),
    "styles_dir": OptionInfo(os.path.join(paths.data_path, 'styles.csv'), "File or Folder with user-defined styles", folder=True),
@@ -516,7 +516,7 @@ options_templates.update(options_section(('system-paths', "System Paths"), {

 options_templates.update(options_section(('saving-images', "Image Options"), {
    "samples_save": OptionInfo(True, "Always save all generated images"),
-    "samples_format": OptionInfo('jpg', 'File format for generated images', gr.Dropdown, lambda: {"choices": ["jpg", "png", "webp", "tiff", "jp2"]}),
+    "samples_format": OptionInfo('jpg', 'File format for generated images', gr.Dropdown, {"choices": ["jpg", "png", "webp", "tiff", "jp2"]}),
    "jpeg_quality": OptionInfo(90, "Quality for saved jpeg images", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}),
    "img_max_size_mp": OptionInfo(250, "Maximum image size (MP)", gr.Slider, {"minimum": 100, "maximum": 2000, "step": 1}),
    "webp_lossless": OptionInfo(False, "Use lossless compression for webp images"),
@@ -531,7 +531,7 @@ options_templates.update(options_section(('saving-images', "Image Options"), {
    "image_watermark": OptionInfo('', "Image watermark string"),
    "image_sep_grid": OptionInfo("<h2>Grid Options</h2>", "", gr.HTML),
    "grid_save": OptionInfo(True, "Always save all generated image grids"),
-    "grid_format": OptionInfo('jpg', 'File format for grids', gr.Dropdown, lambda: {"choices": ["jpg", "png", "webp", "tiff", "jp2"]}),
+    "grid_format": OptionInfo('jpg', 'File format for grids', gr.Dropdown, {"choices": ["jpg", "png", "webp", "tiff", "jp2"]}),
    "n_rows": OptionInfo(-1, "Grid row count", gr.Slider, {"minimum": -1, "maximum": 16, "step": 1}),

    "save_sep_options": OptionInfo("<h2>Intermediate Image Saving</h2>", "", gr.HTML),
@@ -572,6 +572,7 @@ options_templates.update(options_section(('saving-paths', "Image Naming & Paths"
 }))

 options_templates.update(options_section(('ui', "User Interface"), {
+    "motd": OptionInfo(True, "Show MOTD"),
    "gradio_theme": OptionInfo("black-teal", "UI theme", gr.Dropdown, lambda: {"choices": list_themes()}, refresh=refresh_themes),
    "theme_style": OptionInfo("Auto", "Theme mode", gr.Radio, {"choices": ["Auto", "Dark", "Light"]}),
    "tooltips": OptionInfo("UI Tooltips", "UI tooltips", gr.Radio, {"choices": ["None", "Browser default", "UI tooltips"], "visible": False}),
@@ -617,12 +618,12 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"),
    "schedulers_use_karras": OptionInfo(True, "Use Karras sigmas", gr.Checkbox, {"visible": False}),
    "schedulers_use_thresholding": OptionInfo(False, "Use dynamic thresholding", gr.Checkbox, {"visible": False}),
    "schedulers_use_loworder": OptionInfo(True, "Use simplified solvers in final steps", gr.Checkbox, {"visible": False}),
-    "schedulers_prediction_type": OptionInfo("default", "Override model prediction type", gr.Radio, lambda: {"choices": ['default', 'epsilon', 'sample', 'v-prediction'], "visible": False}),
+    "schedulers_prediction_type": OptionInfo("default", "Override model prediction type", gr.Radio, {"choices": ['default', 'epsilon', 'sample', 'v-prediction'], "visible": False}),

    # managed from ui.py for backend diffusers
    "schedulers_sep_diffusers": OptionInfo("<h2>Diffusers specific config</h2>", "", gr.HTML),
-    "schedulers_dpm_solver": OptionInfo("sde-dpmsolver++", "DPM solver algorithm", gr.Radio, lambda: {"choices": ['dpmsolver', 'dpmsolver++', 'sde-dpmsolver', 'sde-dpmsolver++']}),
-    "schedulers_beta_schedule": OptionInfo("default", "Beta schedule", gr.Radio, lambda: {"choices": ['default', 'linear', 'scaled_linear', 'squaredcos_cap_v2']}),
+    "schedulers_dpm_solver": OptionInfo("sde-dpmsolver++", "DPM solver algorithm", gr.Radio, {"choices": ['dpmsolver', 'dpmsolver++', 'sde-dpmsolver', 'sde-dpmsolver++']}),
+    "schedulers_beta_schedule": OptionInfo("default", "Beta schedule", gr.Radio, {"choices": ['default', 'linear', 'scaled_linear', 'squaredcos_cap_v2']}),
    'schedulers_beta_start': OptionInfo(0, "Beta start", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.00001}),
    'schedulers_beta_end': OptionInfo(0, "Beta end", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.00001}),

@@ -640,6 +641,8 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"),
    'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}),
    'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}),
    "ddim_discretize": OptionInfo('uniform', "DDIM discretize img2img", gr.Radio, {"choices": ['uniform', 'quad']}),
+    "pad_cond_uncond": OptionInfo(True, "Pad prompt and negative prompt to be same length", gr.Checkbox, {"visible": False}), # TODO implementation missing
+    "batch_cond_uncond": OptionInfo(True, "Do conditional and unconditional denoising in one batch", gr.Checkbox, {"visible": False}), # TODO implementation missing
 }))

 options_templates.update(options_section(('postprocessing', "Postprocessing"), {
@@ -697,19 +700,28 @@ options_templates.update(options_section(('interrogate', "Interrogate"), {
 }))

 options_templates.update(options_section(('extra_networks', "Extra Networks"), {
+    "extra_networks_sep1": OptionInfo("<h2>Extra networks UI</h2>", "", gr.HTML),
    "extra_networks": OptionInfo(["All"], "Extra networks", ui_components.DropdownMulti, lambda: {"choices": ['All'] + [en.title for en in extra_networks]}),
-    "extra_networks_styles": OptionInfo(True, "Show built-in styles"),
-    "extra_networks_card_cover": OptionInfo("sidebar", "UI position", gr.Radio, lambda: {"choices": ["cover", "inline", "sidebar"]}),
+    "extra_networks_card_cover": OptionInfo("sidebar", "UI position", gr.Radio, {"choices": ["cover", "inline", "sidebar"]}),
    "extra_networks_height": OptionInfo(53, "UI height (%)", gr.Slider, {"minimum": 10, "maximum": 100, "step": 1}),
    "extra_networks_sidebar_width": OptionInfo(35, "UI sidebar width (%)", gr.Slider, {"minimum": 10, "maximum": 80, "step": 1}),
    "extra_networks_card_size": OptionInfo(160, "UI card size (px)", gr.Slider, {"minimum": 20, "maximum": 2000, "step": 1}),
    "extra_networks_card_square": OptionInfo(True, "UI disable variable aspect ratio"),
-    "extra_networks_card_fit": OptionInfo("cover", "UI image contain method", gr.Radio, lambda: {"choices": ["contain", "cover", "fill"], "visible": False}),
+    "extra_networks_card_fit": OptionInfo("cover", "UI image contain method", gr.Radio, {"choices": ["contain", "cover", "fill"], "visible": False}),
+
+    "extra_networks_sep2": OptionInfo("<h2>Extra networks general</h2>", "", gr.HTML),
    "extra_network_skip_indexing": OptionInfo(False, "Do not automatically build extra network pages", gr.Checkbox),
+    "extra_networks_default_multiplier": OptionInfo(1.0, "Default multiplier for extra networks", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),
+
+    "extra_networks_sep3": OptionInfo("<h2>Extra networks settings</h2>", "", gr.HTML),
+    "extra_networks_styles": OptionInfo(True, "Show built-in styles"),
+    "lora_preferred_name": OptionInfo("filename", "LoRA preffered name", gr.Radio, {"choices": ["filename", "alias"]}),
+    "lora_add_hashes_to_infotext": OptionInfo(True, "LoRA add hash info"),
+    "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}),
    "lyco_patch_lora": OptionInfo(False, "Use LyCoris handler for all LoRA types", gr.Checkbox, { "visible": False }),
    "lora_functional": OptionInfo(False, "Use Kohya method for handling multiple LoRA", gr.Checkbox, { "visible": False }),
-    "extra_networks_default_multiplier": OptionInfo(1.0, "Default multiplier for extra networks", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),
-    "sd_hypernetwork": OptionInfo("None", "Add hypernetwork to prompt", gr.Dropdown, lambda: { "choices": ["None"] + list(hypernetworks.keys()), "visible": False }, refresh=reload_hypernetworks),
+
+    "sd_hypernetwork": OptionInfo("None", "Add hypernetwork to prompt", gr.Dropdown, { "choices": ["None"], "visible": False }),
 }))

 options_templates.update(options_section((None, "Hidden options"), {
@@ -915,7 +927,7 @@ def reload_gradio_theme(theme_name=None):
            'font_mono':['IBM Plex Mono', 'ui-monospace', 'Consolas', 'monospace']
        }
    if theme_name in list_builtin_themes():
-        gradio_theme = gr.themes.Default(**default_font_params)
+        gradio_theme = gr.themes.Base(**default_font_params)
    elif theme_name.startswith("gradio/"):
        if theme_name == "gradio/default":
            gradio_theme = gr.themes.Default(**default_font_params)
@@ -60,7 +60,7 @@ def apply_styles_to_extra(p, style: Style):
                v = type(orig)(v)
            setattr(p, k, v)
            fields.append(f'{k}={v}')
-    log.debug(f'Applied style: {style.name} extra={fields}')
+    log.info(f'Applying style: name={style.name} extra={fields}')


 class StyleDatabase:
@@ -543,7 +543,6 @@ def create_ui(startup_timer = None):
            negative_token_button.click(fn=wrap_queued_call(update_token_counter), inputs=[txt2img_negative_prompt, steps], outputs=[negative_token_counter])

            ui_extra_networks.setup_ui(extra_networks_ui, txt2img_gallery)
-            # log.debug(f'UI interface: tab=txt2img batch={show_batch.value} seed={show_seed.value} advanced={show_advanced.value} second_pass={show_second_pass.value}')

    timer.startup.record("ui-txt2img")

@@ -579,19 +578,19 @@ def create_ui(startup_timer = None):
                with gr.Tabs(elem_id="mode_img2img"):
                    img2img_selected_tab = gr.State(0) # pylint: disable=abstract-class-instantiated
                    with gr.TabItem('Image', id='img2img', elem_id="img2img_img2img_tab") as tab_img2img:
-                        init_img = gr.Image(label="Image for img2img", elem_id="img2img_image", show_label=False, source="upload", interactive=True, type="pil", tool="editor", image_mode="RGBA", height=480)
+                        init_img = gr.Image(label="Image for img2img", elem_id="img2img_image", show_label=False, source="upload", interactive=True, type="pil", tool="editor", image_mode="RGBA", height=512)
                        add_copy_image_controls('img2img', init_img)

                    with gr.TabItem('Sketch', id='img2img_sketch', elem_id="img2img_img2img_sketch_tab") as tab_sketch:
-                        sketch = gr.Image(label="Image for img2img", elem_id="img2img_sketch", show_label=False, source="upload", interactive=True, type="pil", tool="color-sketch", image_mode="RGBA", height=480)
+                        sketch = gr.Image(label="Image for img2img", elem_id="img2img_sketch", show_label=False, source="upload", interactive=True, type="pil", tool="color-sketch", image_mode="RGBA", height=512)
                        add_copy_image_controls('sketch', sketch)

                    with gr.TabItem('Inpaint', id='inpaint', elem_id="img2img_inpaint_tab") as tab_inpaint:
-                        init_img_with_mask = gr.Image(label="Image for inpainting with mask", show_label=False, elem_id="img2maskimg", source="upload", interactive=True, type="pil", tool="sketch", image_mode="RGBA", height=480)
+                        init_img_with_mask = gr.Image(label="Image for inpainting with mask", show_label=False, elem_id="img2maskimg", source="upload", interactive=True, type="pil", tool="sketch", image_mode="RGBA", height=512)
                        add_copy_image_controls('inpaint', init_img_with_mask)

                    with gr.TabItem('Inpaint sketch', id='inpaint_sketch', elem_id="img2img_inpaint_sketch_tab") as tab_inpaint_color:
-                        inpaint_color_sketch = gr.Image(label="Color sketch inpainting", show_label=False, elem_id="inpaint_sketch", source="upload", interactive=True, type="pil", tool="color-sketch", image_mode="RGBA", height=480)
+                        inpaint_color_sketch = gr.Image(label="Color sketch inpainting", show_label=False, elem_id="inpaint_sketch", source="upload", interactive=True, type="pil", tool="color-sketch", image_mode="RGBA", height=512)
                        inpaint_color_sketch_orig = gr.State(None) # pylint: disable=abstract-class-instantiated
                        add_copy_image_controls('inpaint_sketch', inpaint_color_sketch)

@@ -674,9 +673,6 @@ def create_ui(startup_timer = None):
                                    scale_by.release(**on_change_args)
                                    button_update_resize_to.click(**on_change_args)

-                                    # the code below is meant to update the resolution label after the image in the image selection UI has changed.
-                                    # as it is now the event keeps firing continuously for inpaint edits, which ruins the page with constant requests.
-                                    # I assume this must be a gradio bug and for now we'll just do it for non-inpaint inputs.
                                    for component in [init_img, sketch]:
                                        component.change(fn=lambda: None, _js="updateImg2imgResizeToTextAfterChangingImage", inputs=[], outputs=[], show_progress=False)

@@ -946,8 +942,8 @@ def create_ui(startup_timer = None):
        def get_opt_values():
            return [getattr(opts, _key) for _key in keys_to_reset]

-        elements_to_reset = [component_dict[_key] for _key in keys_to_reset]
-        indicator = gr.Button("", elem_classes="modification-indicator", elem_id="modification_indicator_" + key, **kwargs)
+        elements_to_reset = [component_dict[_key] for _key in keys_to_reset if component_dict[_key] is not None]
+        indicator = gr.Button("", elem_classes="modification-indicator", elem_id=f"modification_indicator_{key}", **kwargs)
        indicator.click(fn=get_opt_values, outputs=elements_to_reset, show_progress=False)
        return indicator

@@ -1135,6 +1131,7 @@ def create_ui(startup_timer = None):
            gr.Audio(interactive=False, value=os.path.join(script_path, opts.notification_audio_path), elem_id="audio_notification", visible=False)

        text_settings = gr.Textbox(elem_id="settings_json", value=lambda: opts.dumpjson(), visible=False)
+        components = [c for c in components if c is not None]
        settings_submit.click(
            fn=wrap_gradio_call(run_settings, extra_outputs=[gr.update()]),
            inputs=components,
@@ -1185,7 +1182,7 @@ def create_ui(startup_timer = None):
        demo.load(
            fn=get_settings_values,
            inputs=[],
-            outputs=[component_dict[k] for k in component_keys],
+            outputs=[component_dict[k] for k in component_keys if component_dict[k] is not None],
            queue=False,
        )

@@ -72,10 +72,14 @@ class UiLoadsave:
            apply_field(x, 'value')
        if type(x) == gr.Dropdown:
            def check_dropdown(val):
+                if x.choices is None:
+                    errors.log.warning(f'UI: path={path} value={getattr(x, "value", None)}, choices={getattr(x, "choices", None)}')
+                    return False
+                choices = [c[0] for c in x.choices] if type(x.choices[0]) == tuple else x.choices
                if getattr(x, 'multiselect', False):
-                    return all(value in x.choices for value in val)
+                    return all(value in choices for value in val)
                else:
-                    return val in x.choices
+                    return val in choices
            apply_field(x, 'value', check_dropdown, getattr(x, 'init_field', None))

        def check_tab_id(tab_id):
@@ -312,12 +312,16 @@ def create_ui():
                            for variant in model['modelVersions']:
                                if variant['id'] == variant_id:
                                    for f in variant['files']:
-                                        data3.append([
-                                            f['name'],
-                                            round(f['sizeKB']),
-                                            json.dumps(f['metadata']),
-                                            f['downloadUrl'],
-                                        ])
+                                        try:
+                                            if os.path.splitext(f['name'])[1].lower() in ['.safetensors', '.ckpt', '.pt', '.pth', '.bin']:
+                                                data3.append([
+                                                    f['name'],
+                                                    round(f['sizeKB']),
+                                                    json.dumps(f['metadata']),
+                                                    f['downloadUrl'],
+                                                ])
+                                        except Exception:
+                                            pass
                    log.debug(f'CivitAI select: model="{in_data[evt.index[0]]}" files={len(data3)}')
                    return data3

@@ -0,0 +1 @@
+ 
@@ -51,7 +51,7 @@ accelerate==0.20.3
 opencv-python-headless==4.7.0.72
 diffusers==0.21.4
 einops==0.4.1
-gradio==3.43.2
+gradio==3.44.4
 huggingface_hub==0.17.1
 numexpr==2.8.4
 numpy==1.24.4