Merge pull request #2490 from vladmandic/master

update dev
2026-09-18 16:54:33 +02:00 · 2023-11-10 10:13:19 -05:00
parent f69759c218 056b04dcd0
commit c5fde5d7da
56 changed files with 754 additions and 957 deletions
@@ -1,34 +1,36 @@
 [submodule "wiki"]
  path = wiki
  url = https://github.com/vladmandic/automatic.wiki
-[submodule "extensions-builtin/sd-extension-system-info"]
-  path = extensions-builtin/sd-extension-system-info
-  url = https://github.com/vladmandic/sd-extension-system-info
-[submodule "extensions-builtin/stable-diffusion-webui-images-browser"]
-  path = extensions-builtin/stable-diffusion-webui-images-browser
-  url = https://github.com/AlUlkesh/stable-diffusion-webui-images-browser
-  ignore = dirty
-[submodule "modules/lora"]
-  path = modules/lora
-  url = https://github.com/kohya-ss/sd-scripts
-  ignore = dirty
-[submodule "extensions-builtin/sd-webui-controlnet"]
-  path = extensions-builtin/sd-webui-controlnet
-  url = https://github.com/Mikubill/sd-webui-controlnet
-  ignore = dirty
-[submodule "extensions-builtin/stable-diffusion-webui-rembg"]
-  path = extensions-builtin/stable-diffusion-webui-rembg
-  url = https://github.com/vladmandic/sd-extension-rembg
-  ignore = dirty
-[submodule "extensions-builtin/sd-webui-agent-scheduler"]
-  path = extensions-builtin/sd-webui-agent-scheduler
-  url = https://github.com/ArtVentureX/sd-webui-agent-scheduler
-  ignore = dirty
-[submodule "extensions-builtin/sd-extension-chainner"]
-  path = extensions-builtin/sd-extension-chainner
-  url = https://github.com/vladmandic/sd-extension-chainner
  ignore = dirty
 [submodule "modules/k-diffusion"]
  path = modules/k-diffusion
  url = https://github.com/crowsonkb/k-diffusion
  ignore = dirty
+[submodule "modules/lora"]
+  path = modules/lora
+  url = https://github.com/kohya-ss/sd-scripts
+  ignore = dirty
+[submodule "extensions-builtin/sd-extension-system-info"]
+  path = extensions-builtin/sd-extension-system-info
+  url = https://github.com/vladmandic/sd-extension-system-info
+  ignore = dirty
+[submodule "extensions-builtin/sd-extension-chainner"]
+  path = extensions-builtin/sd-extension-chainner
+  url = https://github.com/vladmandic/sd-extension-chainner
+  ignore = dirty
+[submodule "extensions-builtin/stable-diffusion-webui-rembg"]
+  path = extensions-builtin/stable-diffusion-webui-rembg
+  url = https://github.com/vladmandic/sd-extension-rembg
+  ignore = dirty
+[submodule "extensions-builtin/stable-diffusion-webui-images-browser"]
+  path = extensions-builtin/stable-diffusion-webui-images-browser
+  url = https://github.com/AlUlkesh/stable-diffusion-webui-images-browser
+  ignore = dirty
+[submodule "extensions-builtin/sd-webui-controlnet"]
+  path = extensions-builtin/sd-webui-controlnet
+  url = https://github.com/Mikubill/sd-webui-controlnet
+  ignore = dirty
+[submodule "extensions-builtin/sd-webui-agent-scheduler"]
+  path = extensions-builtin/sd-webui-agent-scheduler
+  url = https://github.com/ArtVentureX/sd-webui-agent-scheduler
+  ignore = dirty
@@ -1,12 +1,47 @@
 # Change Log for SD.Next

+## Update for 2023-11-10
+
+- **Diffusers**  
+  - **LCM** support for any *SD 1.5* or *SD-XL* model!  
+    - download [lcm-lora-sd15](https://huggingface.co/latent-consistency/lcm-lora-sdv1-5/tree/main) and/or [lcm-lora-sdxl](https://huggingface.co/latent-consistency/lcm-lora-sdxl/tree/main)  
+    - load for favorite *SD 1.5* or *SD-XL* model *(original LCM was SD 1.5 only, this is both)*  
+    - load **lcm lora**  
+    - set **sampler** to **LCM**  
+    - set number of steps to some low number, for SD-XL 6-7 steps is normally sufficient  
+      note: LCM scheduler does not support steps higher than 50
+    - set cfg to 1 or 2  
+  - Add `cli/lcm-convert.py` script to convert any SD 1.5 or SD-XL model to LCM model  
+    by baking in LORA and uploading to Huggingface, thanks @Disty0  
+  - Add additional pipeline types for manual model loads when loading from `safetensors`  
+  - Updated logic for calculating **steps** when using base/hires/refiner workflows  
+  - Safe model offloading for non-standard models  
+  - Fix **DPM SDE** scheduler  
+  - Update to `diffusers==0.23.0`  
+- **Extra networks**  
+  - Use multi-threading for 5x load speedup  
+- **General**:  
+  - Reworked parser when pasting previously generated images/prompts  
+    includes all `txt2img`, `img2img` and `override` params  
+  - Add refiner options to XYZ Grid  
+  - Support custom upscalers in subfolders  
+  - Support `--ckpt none` to skip loading a model  
+- **Fixes**  
+  - Fix inpaint  
+  - Fix manual grid image save  
+  - Fix img2img init image save  
+  - More uniform models paths  
+  - Safe scripts callback execution  
+  - Improve extension compatibility  
+  - Improve BF16 support  
+
 ## Update for 2023-11-06

 Another pretty big release, this time with focus on new models (3 new model types), new backends and optimizations
 Plus quite a few fixes  

 Also, [Wiki](https://github.com/vladmandic/automatic/wiki) has been updated with new content, so check it out!  
-Some highlights: [OpenVINO](https://github.com/vladmandic/automatic/wiki/OpenVINO), [IntelArc](https://github.com/vladmandic/automatic/wiki/Intel-ARC), [DirectML](https://github.com/vladmandic/automatic/wiki/DirectML), [ONNX/Olive>](https://github.com/vladmandic/automatic/wiki/ONNX-Runtime)
+Some highlights: [OpenVINO](https://github.com/vladmandic/automatic/wiki/OpenVINO), [IntelArc](https://github.com/vladmandic/automatic/wiki/Intel-ARC), [DirectML](https://github.com/vladmandic/automatic/wiki/DirectML), [ONNX/Olive](https://github.com/vladmandic/automatic/wiki/ONNX-Olive)

 - **Diffusers**
  - since now **SD.Next** supports **12** different model types, we've added reference model for each type in  
@@ -176,7 +176,7 @@ General goals:
 ### **Sponsors**

 <div align="center">
-<!-- sponsors --><a href="https://github.com/allangrant"><img src="https://github.com/allangrant.png" width="60px" alt="Allan Grant" /></a><a href="https://github.com/demib72"><img src="https://github.com/demib72.png" width="60px" alt="Michael Harris" /></a><a href="https://github.com/BrentOzar"><img src="https://github.com/BrentOzar.png" width="60px" alt="Brent Ozar" /></a><a href="https://github.com/AimwiseTonix"><img src="https://github.com/AimwiseTonix.png" width="60px" alt="ToniX" /></a><a href="https://github.com/inktomi"><img src="https://github.com/inktomi.png" width="60px" alt="Matthew Runo" /></a><a href="https://github.com/HELLO-WORLD-SAS"><img src="https://github.com/HELLO-WORLD-SAS.png" width="60px" alt="HELLO WORLD SAS" /></a><a href="https://github.com/4joeknight4"><img src="https://github.com/4joeknight4.png" width="60px" alt="" /></a><a href="https://github.com/SaladTechnologies"><img src="https://github.com/SaladTechnologies.png" width="60px" alt="Salad Technologies" /></a><a href="https://github.com/gymdreams8"><img src="https://github.com/gymdreams8.png" width="60px" alt="Gym Dreams • GymDreams8" /></a><a href="https://github.com/mantzaris"><img src="https://github.com/mantzaris.png" width="60px" alt="a.v.mantzaris" /></a><a href="https://github.com/anothertal3"><img src="https://github.com/anothertal3.png" width="60px" alt="" /></a><!-- sponsors -->
+<!-- sponsors --><a href="https://github.com/allangrant"><img src="https://github.com/allangrant.png" width="60px" alt="Allan Grant" /></a><a href="https://github.com/demib72"><img src="https://github.com/demib72.png" width="60px" alt="Michael Harris" /></a><a href="https://github.com/BrentOzar"><img src="https://github.com/BrentOzar.png" width="60px" alt="Brent Ozar" /></a><a href="https://github.com/AimwiseTonix"><img src="https://github.com/AimwiseTonix.png" width="60px" alt="ToniX" /></a><a href="https://github.com/inktomi"><img src="https://github.com/inktomi.png" width="60px" alt="Matthew Runo" /></a><a href="https://github.com/HELLO-WORLD-SAS"><img src="https://github.com/HELLO-WORLD-SAS.png" width="60px" alt="HELLO WORLD SAS" /></a><a href="https://github.com/4joeknight4"><img src="https://github.com/4joeknight4.png" width="60px" alt="" /></a><a href="https://github.com/SaladTechnologies"><img src="https://github.com/SaladTechnologies.png" width="60px" alt="Salad Technologies" /></a><a href="https://github.com/mantzaris"><img src="https://github.com/mantzaris.png" width="60px" alt="a.v.mantzaris" /></a><a href="https://github.com/anothertal3"><img src="https://github.com/anothertal3.png" width="60px" alt="" /></a><!-- sponsors -->
 </div>

 <br>
@@ -0,0 +1,55 @@
+import os
+import argparse
+import torch
+from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, AutoPipelineForText2Image, LCMScheduler
+
+parser = argparse.ArgumentParser("lcm_convert")
+parser.add_argument("--name", help="Name of the new LCM model", type=str)
+parser.add_argument("--model", help="A model to convert", type=str)
+parser.add_argument("--lora-scale", default=1.0, help="Strenght of the LCM", type=float)
+parser.add_argument("--huggingface", action="store_true", help="Use Hugging Face models instead of safetensors models")
+parser.add_argument("--upload", action="store_true", help="Upload the new LCM model to Hugging Face")
+parser.add_argument("--no-half", action="store_true", help="Convert the new LCM model to FP32")
+parser.add_argument("--no-save", action="store_true", help="Don't save the new LCM model to local disk")
+parser.add_argument("--sdxl", action="store_true", help="Use SDXL models")
+parser.add_argument("--ssd-1b", action="store_true", help="Use SSD-1B models")
+
+args = parser.parse_args()
+
+if args.huggingface:
+    pipeline = AutoPipelineForText2Image.from_pretrained(args.model, torch_dtype=torch.float16, variant="fp16")
+else:
+    if args.sdxl or args.ssd_1b:
+        pipeline = StableDiffusionXLPipeline.from_single_file(args.model)
+    else:
+        pipeline = StableDiffusionPipeline.from_single_file(args.model)
+
+pipeline.scheduler = LCMScheduler.from_config(pipeline.scheduler.config)
+if args.sdxl:
+    pipeline.load_lora_weights("latent-consistency/lcm-lora-sdxl")
+elif args.ssd_1b:
+    pipeline.load_lora_weights("latent-consistency/lcm-lora-ssd-1b")
+else:
+    pipeline.load_lora_weights("latent-consistency/lcm-lora-sdv1-5")
+pipeline.fuse_lora(lora_scale=args.lora_scale)
+
+#components = pipeline.components
+#pipeline = LatentConsistencyModelPipeline(**components)
+
+if args.no_half:
+    pipeline = pipeline.to(dtype=torch.float32)
+else:
+    pipeline = pipeline.to(dtype=torch.float16)
+print(pipeline)
+
+if not args.no_save:
+    os.makedirs(f"models--local--{args.name}/snapshots")
+    if args.no_half:
+        pipeline.save_pretrained(f"models--local--{args.name}/snapshots/{args.name}")
+    else:
+        pipeline.save_pretrained(f"models--local--{args.name}/snapshots/{args.name}", variant="fp16")
+if args.upload:
+    if args.no_half:
+        pipeline.push_to_hub(args.name)
+    else:
+        pipeline.push_to_hub(args.name, variant="fp16")
@@ -1,5 +1,6 @@
 import os
 import json
+import concurrent
 import network
 import networks
 from modules import shared, ui_extra_networks
@@ -8,6 +9,7 @@ from modules import shared, ui_extra_networks
 class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
    def __init__(self):
        super().__init__('Lora')
+        self.list_time = 0

    def refresh(self):
        networks.list_available_networks()
@@ -74,10 +76,12 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
            return None

    def list_items(self):
-        for _index, name in enumerate(networks.available_networks):
-            item = self.create_item(name)
-            if item is not None:
-                yield item
+        with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
+            future_items = {executor.submit(self.create_item, net): net for net in networks.available_networks}
+            for future in concurrent.futures.as_completed(future_items):
+                item = future.result()
+                if item is not None:
+                    yield item

    def allowed_directories_for_previews(self):
        return [shared.cmd_opts.lora_dir, shared.cmd_opts.lyco_dir]
@@ -61,7 +61,7 @@
  {"id":"","label":"Skip","localized":"","hint":"Stop processing current job and continue processing"},
  {"id":"","label":"Interrupt","localized":"","hint":"Interrupt current processing job"},
  {"id":"","label":"Pause","localized":"","hint":"Pause processing"},
-  {"id":"","label":"Restore","localized":"","hint":"Restore parameters from last known generated image"},
+  {"id":"","label":"Restore","localized":"","hint":"Restore parameters from current prompt or last known generated image"},
  {"id":"","label":"Clear","localized":"","hint":"Clear prompts"},
  {"id":"","label":"Networks","localized":"","hint":"Open extra network interface"},
  {"id":"","label":"Interrogate\nCLIP","localized":"","hint":"Run interrogate using CLIP model"},
@@ -24,7 +24,12 @@
    "desc": "Segmind's Tiny-SD offers a compact, efficient, and distilled version of Realistic Vision 4.0 and is up to 80% faster than SD1.5",
    "preview": "segmind--tiny-sd.jpg"
  },
-  "LCM Dreamshaper 7": {
+    "LCM SD-XL": {
+    "path": "latent-consistency/lcm-sdxl",
+    "desc": "Latent Consistencey Models enable swift inference with minimal steps on any pre-trained LDMs, including Stable Diffusion. By distilling classifier-free guidance into the model's input, LCM can generate high-quality images in very short inference time. LCM can generate quality images in as few as 3-4 steps, making it blazingly fast.",
+    "preview": "latent-consistency--lcm-sdxl.jpg"
+  },
+  "LCM SD-1.5 Dreamshaper 7": {
    "path": "SimianLuo/LCM_Dreamshaper_v7",
    "desc": "Latent Consistencey Models enable swift inference with minimal steps on any pre-trained LDMs, including Stable Diffusion. By distilling classifier-free guidance into the model's input, LCM can generate high-quality images in very short inference time. LCM can generate quality images in as few as 3-4 steps, making it blazingly fast.",
    "preview": "simianluo--lcm_dreamshaper_v7.jpg"
@@ -201,7 +201,7 @@ def installed(package, friendly: str = None, reload = False, quiet = False):


 def uninstall(package):
-    if installed(package, package):
+    if installed(package, package, quiet=True):
        log.warning(f'Uninstalling: {package}')
        pip(f"uninstall {package} --yes --quiet", ignore=True, quiet=True)

@@ -38,7 +38,7 @@ html { font-size: var(--font-size); font-family: var(--font); }
 body, button, input, select, textarea { font-family: var(--font); }
 button { font-size: 1.2rem; max-width: 400px; }
 img { background-color: var(--background-color); }
-input[type=range] { height: var(--line-sm) !important; appearance: none !important; margin-top: 0 !important; min-width: 100% !important;
+input[type=range] { height: var(--line-sm) !important; appearance: none !important; margin-top: 0 !important; min-width: max(4em, 100%) !important;
  background-color: var(--background-color) !important; width: 100% !important; background: transparent !important; }
 input[type=range]::-webkit-slider-runnable-track { width: 100% !important; height: var(--line-sm) !important; cursor: pointer !important; box-shadow: 2px 2px 3px #111111 !important;
  background: var(--input-background-fill) !important; border-radius: var(--radius-lg) !important; border: 0px solid #222222 !important; }
@@ -40,6 +40,7 @@ const setENState = (state) => {
 // methods

 function showCardDetails(event) {
+  console.log('showCardDetails', event)
  const tabname = getENActiveTab();
  const btn = gradioApp().getElementById(`${tabname}_extra_details_btn`);
  btn.click();
@@ -30,7 +30,7 @@ textarea { overflow-y: auto !important; }
 .gradio-column { min-width: min(160px, 100%) !important; }
 .gradio-container { max-width: unset !important; padding: var(--block-label-padding) !important; }
 .gradio-container .prose a, .gradio-container .prose a:visited{ color: unset; text-decoration: none; }
-.gradio-dropdown { margin-right: var(--spacing-sm) !important; }
+.gradio-dropdown { margin-right: var(--spacing-sm) !important; min-width:160px; max-width:fit-content }
 .gradio-dropdown ul.options { z-index: 1000; min-width: fit-content; max-height: 33vh !important; white-space: nowrap; }
 .gradio-dropdown ul.options li.item { padding: var(--spacing-xs); }
 .gradio-dropdown ul.options li.item:not(:has(.hide)) { background-color: var(--primary-500); }
@@ -106,11 +106,10 @@ div#extras_scale_to_tab div.form{ flex-direction: row; }
 #quicksettings > button { padding: 0 1em 0 0; align-self: end; margin-bottom: var(--text-sm); }
 #settings { display: flex; gap: var(--layout-gap); }
 #settings div { border: none; gap: 0; margin: 0 0 var(--layout-gap) 0px; padding: 0; }
-#settings .gr-group { max-width: 70em; }
 #settings > div.tab-content { flex: 10 0 75%; display: grid; }
 #settings > div.tab-content > div { border: none; padding: 0; }
 #settings > div.tab-content > div > div > div > div > div { flex-direction: unset; }
-#settings > div.tab-nav { display: grid; grid-template-columns: repeat(auto-fill, .5em minmax(10em, 1fr)); flex: 1 0 auto; width: 12em; align-self: flex-start; gap: var(--spacing-lg); }
+#settings > div.tab-nav { display: grid; grid-template-columns: repeat(auto-fill, .5em minmax(10em, 1fr)); flex: 1 0 auto; width: 12em; align-self: flex-start; gap: var(--spacing-xxl); }
 #settings > div.tab-nav button { display: block; border: none; text-align: left; white-space: initial; padding: 0; }
 #settings > div.tab-nav > #settings_show_all_pages { padding: var(--size-2) var(--size-4); }
 #settings .block.gradio-checkbox { margin: 0; width: auto; }
@@ -77,7 +77,7 @@ function markIfModified(setting_name, value) {
  tab_nav_indicator.classList.toggle('saved', saved.size > 0);
  if (changed_items.size > 0) tab_nav_indicator.title += `click to reset ${changed_items.size} unapplied changes in this tab\n`;
  if (saved.size > 0) tab_nav_indicator.title += `${saved.size} custom values\n${unsaved.size} default values}`;
-  elem.scrollIntoView({ behavior: 'smooth', block: 'center' }); // TODO why is scroll happening on every change if all pages are visible?
+  // elem.scrollIntoView({ behavior: 'smooth', block: 'center' }); // TODO why is scroll happening on every change if all pages are visible?
 }

 onAfterUiUpdate(async () => {
@@ -105,9 +105,10 @@ onAfterUiUpdate(async () => {
    },
  });

-  const settings_search = gradioApp().querySelectorAll('#settings_search > label > textarea')[0];
-  settings_search.oninput = (e) => {
+  const settingsSearch = gradioApp().querySelectorAll('#settings_search > label > textarea')[0];
+  settingsSearch.oninput = (e) => {
    setTimeout(() => {
+      log('settingsSearch', e.target.value)
      showAllSettings();
      gradioApp().querySelectorAll('#tab_settings .tabitem').forEach((section) => {
        section.querySelectorAll('.dirtyable').forEach((setting) => {
@@ -131,23 +132,23 @@ onOptionsChanged(() => {
 function initSettings() {
  if (settingsInitialized) return;
  settingsInitialized = true;
-  const tab_nav_element = gradioApp().querySelector('#settings > .tab-nav');
-  const tab_nav_buttons = gradioApp().querySelectorAll('#settings > .tab-nav > button');
-  const tab_elements = gradioApp().querySelectorAll('#settings > div:not(.tab-nav)');
+  const tabNavElements = gradioApp().querySelector('#settings > .tab-nav');
+  const tabNavButtons = gradioApp().querySelectorAll('#settings > .tab-nav > button');
+  const tabElements = gradioApp().querySelectorAll('#settings > div:not(.tab-nav)');
  const observer = new MutationObserver((mutations) => {
-    const show_all_pages_dummy = gradioApp().getElementById('settings_show_all_pages');
-    if (show_all_pages_dummy.style.display === 'none') { return; }
-    const mutation_on_style = (mut) => mut.type === 'attributes' && mut.attributeName === 'style';
-    if (mutations.some(mutation_on_style)) showAllSettings();
+    const showAllPages = gradioApp().getElementById('settings_show_all_pages');
+    if (showAllPages.style.display === 'none') return;
+    const mutation = (mut) => mut.type === 'attributes' && mut.attributeName === 'style'
+    if (mutations.some(mutation)) showAllSettings();
  });
-  const tab_content_wrapper = document.createElement('div');
-  tab_content_wrapper.className = 'tab-content';
-  tab_nav_element.parentElement.insertBefore(tab_content_wrapper, tab_nav_element.nextSibling);
-  tab_elements.forEach((elem, index) => {
-    const tab_name = elem.id.replace('settings_', '');
-    const indicator = gradioApp().getElementById(`modification_indicator_${tab_name}`);
-    tab_nav_element.insertBefore(indicator, tab_nav_buttons[index]);
-    tab_content_wrapper.appendChild(elem);
+  const tabContentWrapper = document.createElement('div');
+  tabContentWrapper.className = 'tab-content';
+  tabNavElements.parentElement.insertBefore(tabContentWrapper, tabNavElements.nextSibling);
+  tabElements.forEach((elem, index) => {
+    const tabName = elem.id.replace('settings_', '');
+    const indicator = gradioApp().getElementById(`modification_indicator_${tabName}`);
+    tabNavElements.insertBefore(indicator, tabNavButtons[index]);
+    tabContentWrapper.appendChild(elem);
    observer.observe(elem, { attributes: true, attributeFilter: ['style'] });
  });
  log('initSettings');
@@ -170,7 +170,7 @@ function submit_postprocessing(...args) {
  return args;
 }

-const submit = submit_txt2img;
+window.submit = submit_txt2img;

 function modelmerger(...args) {
  const id = randomId();
@@ -16,9 +16,10 @@ class DeepDanbooru:
    def load(self):
        if self.model is not None:
            return
-
+        model_path = os.path.join(paths.models_path, "DeepDanbooru")
+        shared.log.debug(f'Loading interrogate model: type=DeepDanbooru folder={model_path}')
        files = modelloader.load_models(
-            model_path=os.path.join(paths.models_path, "torch_deepdanbooru"),
+            model_path=model_path,
            model_url='https://github.com/AUTOMATIC1111/TorchDeepDanbooru/releases/download/v1/model-resnet_custom_v3.pt',
            ext_filter=[".pt"],
            download_name='model-resnet_custom_v3.pt',
@@ -1,5 +1,8 @@
 import torch
 import diffusers
+import diffusers.utils.torch_utils
+from typing import Optional, Union, Tuple
+

 def PNDMScheduler__get_prev_sample(self, sample: torch.FloatTensor, timestep, prev_timestep, model_output):
    # See formula (9) of PNDM paper https://arxiv.org/pdf/2202.09778.pdf
@@ -45,8 +48,10 @@ def PNDMScheduler__get_prev_sample(self, sample: torch.FloatTensor, timestep, pr

    return prev_sample

+
 diffusers.PNDMScheduler._get_prev_sample = PNDMScheduler__get_prev_sample # pylint: disable=protected-access

+
 def UniPCMultistepScheduler_multistep_uni_p_bh_update(
    self: diffusers.UniPCMultistepScheduler,
    model_output: torch.FloatTensor,
@@ -153,4 +158,103 @@ def UniPCMultistepScheduler_multistep_uni_p_bh_update(
    x_t = x_t.to(x.dtype)
    return x_t

+
 diffusers.UniPCMultistepScheduler.multistep_uni_p_bh_update = UniPCMultistepScheduler_multistep_uni_p_bh_update
+
+
+def LCMScheduler_step(
+        self,
+        model_output: torch.FloatTensor,
+        timestep: int,
+        sample: torch.FloatTensor,
+        generator: Optional[torch.Generator] = None,
+        return_dict: bool = True,
+    ) -> Union[diffusers.schedulers.scheduling_lcm.LCMSchedulerOutput, Tuple]:
+        """
+        Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
+        process from the learned model outputs (most often the predicted noise).
+
+        Args:
+            model_output (`torch.FloatTensor`):
+                The direct output from learned diffusion model.
+            timestep (`float`):
+                The current discrete timestep in the diffusion chain.
+            sample (`torch.FloatTensor`):
+                A current instance of a sample created by the diffusion process.
+            generator (`torch.Generator`, *optional*):
+                A random number generator.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~schedulers.scheduling_lcm.LCMSchedulerOutput`] or `tuple`.
+        Returns:
+            [`~schedulers.scheduling_utils.LCMSchedulerOutput`] or `tuple`:
+                If return_dict is `True`, [`~schedulers.scheduling_lcm.LCMSchedulerOutput`] is returned, otherwise a
+                tuple is returned where the first element is the sample tensor.
+        """
+        if self.num_inference_steps is None:
+            raise ValueError(
+                "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"
+            )
+
+        if self.step_index is None:
+            self._init_step_index(timestep)
+
+        # 1. get previous step value
+        prev_step_index = self.step_index + 1
+        if prev_step_index < len(self.timesteps):
+            prev_timestep = self.timesteps[prev_step_index]
+        else:
+            prev_timestep = timestep
+
+        # 2. compute alphas, betas
+        sample.__str__()
+        alpha_prod_t = self.alphas_cumprod[timestep]
+        alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod
+
+        beta_prod_t = 1 - alpha_prod_t
+        beta_prod_t_prev = 1 - alpha_prod_t_prev
+
+        # 3. Get scalings for boundary conditions
+        c_skip, c_out = self.get_scalings_for_boundary_condition_discrete(timestep)
+
+        # 4. Compute the predicted original sample x_0 based on the model parameterization
+        if self.config.prediction_type == "epsilon":  # noise-prediction
+            predicted_original_sample = (sample - beta_prod_t.sqrt() * model_output) / alpha_prod_t.sqrt()
+        elif self.config.prediction_type == "sample":  # x-prediction
+            predicted_original_sample = model_output
+        elif self.config.prediction_type == "v_prediction":  # v-prediction
+            predicted_original_sample = alpha_prod_t.sqrt() * sample - beta_prod_t.sqrt() * model_output
+        else:
+            raise ValueError(
+                f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` or"
+                " `v_prediction` for `LCMScheduler`."
+            )
+
+        # 5. Clip or threshold "predicted x_0"
+        if self.config.thresholding:
+            predicted_original_sample = self._threshold_sample(predicted_original_sample)
+        elif self.config.clip_sample:
+            predicted_original_sample = predicted_original_sample.clamp(
+                -self.config.clip_sample_range, self.config.clip_sample_range
+            )
+
+        # 6. Denoise model output using boundary conditions
+        denoised = c_out * predicted_original_sample + c_skip * sample
+
+        # 7. Sample and inject noise z ~ N(0, I) for MultiStep Inference
+        # Noise is not used for one-step sampling.
+        if len(self.timesteps) > 1:
+            noise = diffusers.utils.torch_utils.randn_tensor(model_output.shape, generator=generator, device=model_output.device)
+            prev_sample = alpha_prod_t_prev.sqrt() * denoised + beta_prod_t_prev.sqrt() * noise
+        else:
+            prev_sample = denoised
+
+        # upon completion increase step index by one
+        self._step_index += 1
+
+        if not return_dict:
+            return (prev_sample, denoised)
+
+        return diffusers.schedulers.scheduling_lcm.LCMSchedulerOutput(prev_sample=prev_sample, denoised=denoised)
+
+
+diffusers.LCMScheduler.step = LCMScheduler_step
@@ -3,20 +3,20 @@ import io
 import os
 import re
 import json
-
 from PIL import Image
 import gradio as gr
 from modules.paths import data_path
 from modules import shared, ui_tempdir, script_callbacks, images

+
 re_param_code = r'\s*([\w ]+):\s*("(?:\\"[^,]|\\"|\\|[^\"])+"|[^,]*)(?:,|$)'
 re_param = re.compile(re_param_code)
 re_imagesize = re.compile(r"^(\d+)x(\d+)$")
 re_hypernet_hash = re.compile("\(([0-9a-f]+)\)$") # pylint: disable=anomalous-backslash-in-string
 type_of_gr_update = type(gr.update())
-
 paste_fields = {}
 registered_param_bindings = []
+debug = shared.log.info if os.environ.get('SD_PASTE_DEBUG', None) is not None else lambda *args, **kwargs: None


 class ParamBinding:
@@ -203,37 +203,22 @@ def find_hypernetwork_key(hypernet_name, hypernet_hash=None):


 def parse_generation_parameters(x: str):
-    """parses generation parameters string, the one you see in text field under the picture in UI:
-```
-girl with an artist's beret, determined, blue eyes, desert scene, computer monitors, heavy makeup, by Alphonse Mucha and Charlie Bowater, ((eyeshadow)), (coquettish), detailed, intricate
-Negative prompt: ugly, fat, obese, chubby, (((deformed))), [blurry], bad anatomy, disfigured, poorly drawn face, mutation, mutated, (extra_limb), (ugly), (poorly drawn hands), messy drawing
-Steps: 20, Sampler: Euler a, CFG scale: 7, Seed: 965400086, Size: 512x512, Model hash: 45dee52b
-```
-
-    returns a dict with field values
-    """
-    if x is None:
-        return {}
    res = {}
-    prompt = ""
-    negative_prompt = ""
-    done_with_prompt = False
-    *lines, lastline = x.strip().split("\n")
-    if len(re_param.findall(lastline)) < 3:
-        lines.append(lastline)
-        lastline = ''
-    for line in lines:
-        line = line.strip()
-        if line.startswith("Negative prompt:"):
-            done_with_prompt = True
-            line = line[16:].strip()
-        if done_with_prompt:
-            negative_prompt += ("" if negative_prompt == "" else "\n") + line
-        else:
-            prompt += ("" if prompt == "" else "\n") + line
-    res["Prompt"] = prompt
-    res["Negative prompt"] = negative_prompt
-    for k, v in re_param.findall(lastline):
+    if x is None:
+        return res
+    remaining = x.replace('\n', ' ').strip()
+    if len(remaining) == 0:
+        return res
+    remaining = x[7:] if x.startswith('Prompt: ') else x
+    remaining = x[11:] if x.startswith('parameters: ') else x
+    prompt, remaining = remaining.strip().split('Negative prompt: ', maxsplit=1) if 'Negative prompt: ' in remaining else (remaining, '')
+    res["Prompt"] = prompt.strip()
+    negative, remaining = remaining.strip().split('Steps: ', maxsplit=1) if 'Steps: ' in remaining else (remaining, None)
+    res["Negative prompt"] = negative.strip()
+    if remaining is None:
+        return res
+    remaining = f'Steps: {remaining}'
+    for k, v in re_param.findall(remaining.strip()):
        try:
            if v[0] == '"' and v[-1] == '"':
                v = unquote(v)
@@ -245,16 +230,8 @@ Steps: 20, Sampler: Euler a, CFG scale: 7, Seed: 965400086, Size: 512x512, Model
                res[k] = v
        except Exception:
            pass
-
-    # Missing CLIP skip means it was set to 1 (the default)
-    if "Clip skip" not in res:
-        res["Clip skip"] = "1"
-    hypernet = res.get("Hypernet", None)
-    if hypernet is not None:
-        res["Prompt"] += f"""<hypernet:{hypernet}:{res.get("Hypernet strength", "1.0")}>"""
-    if "Hires resize-1" not in res:
-        res["Hires resize-1"] = 0
-        res["Hires resize-2"] = 0
+    res["Full quality"] = res.get('VAE', None) != 'TAESD'
+    debug(f"Parse prompt: {res}")
    return res


@@ -262,7 +239,7 @@ settings_map = {}


 infotext_to_setting_name_mapping = [
-    ('Backed', 'sd_backend'),
+    ('Backend', 'sd_backend'),
    ('Model hash', 'sd_model_checkpoint'),
    ('Refiner', 'sd_model_refiner'),
    ('VAE', 'sd_vae'),
@@ -305,13 +282,6 @@ infotext_to_setting_name_mapping = [


 def create_override_settings_dict(text_pairs):
-    """creates processing's override_settings parameters from gradio's multiselect
-    Example input:
-        ['Clip skip: 2', 'Model hash: e6e99610c4', 'ENSD: 31337']
-
-    Example output:
-        {'CLIP_stop_at_last_layers': 2, 'sd_model_checkpoint': 'e6e99610c4', 'eta_noise_seed_delta': 31337}
-    """
    res = {}
    params = {}
    for pair in text_pairs:
@@ -328,19 +298,20 @@ def create_override_settings_dict(text_pairs):
 def connect_paste(button, local_paste_fields, input_comp, override_settings_component, tabname):

    def paste_func(prompt):
-        if prompt is not None and 'Negative prompt' not in prompt and 'Steps' not in prompt:
-            prompt = None
-        if not prompt and not shared.cmd_opts.hide_ui_dir_config:
+        if prompt is None or len(prompt.strip()) == 0 and not shared.cmd_opts.hide_ui_dir_config:
            filename = os.path.join(data_path, "params.txt")
            if os.path.exists(filename):
                with open(filename, "r", encoding="utf8") as file:
                    prompt = file.read()
+                shared.log.debug(f'Paste prompt: type="params" prompt="{prompt}"')
            else:
                prompt = ''
-        shared.log.debug(f'Paste prompt: {prompt}')
+        else:
+            shared.log.debug(f'Paste prompt: type="current" prompt="{prompt}"')
        params = parse_generation_parameters(prompt)
        script_callbacks.infotext_pasted_callback(prompt, params)
        res = []
+        applied = {}
        for output, key in local_paste_fields:
            if callable(key):
                v = key(params)
@@ -350,6 +321,7 @@ def connect_paste(button, local_paste_fields, input_comp, override_settings_comp
                res.append(gr.update())
            elif isinstance(v, type_of_gr_update):
                res.append(v)
+                applied[key] = v
            else:
                try:
                    valtype = type(output.value)
@@ -358,8 +330,10 @@ def connect_paste(button, local_paste_fields, input_comp, override_settings_comp
                    else:
                        val = valtype(v)
                    res.append(gr.update(value=val))
+                    applied[key] = val
                except Exception:
                    res.append(gr.update())
+        debug(f"Parse apply: {applied}")
        return res

    if override_settings_component is not None:
@@ -380,6 +354,7 @@ def connect_paste(button, local_paste_fields, input_comp, override_settings_comp
                    continue
                vals[param_name] = v
            vals_pairs = [f"{k}: {v}" for k, v in vals.items()]
+            shared.log.debug(f'Settings overrides: {vals_pairs}')
            return gr.Dropdown.update(value=vals_pairs, choices=vals_pairs, visible=len(vals_pairs) > 0)
        local_paste_fields = local_paste_fields + [(override_settings_component, paste_settings)]

@@ -532,6 +532,8 @@ def atomically_save_image():
            file.write(exifinfo)
        if shared.opts.save_log_fn != '' and len(exifinfo) > 0:
            fn = os.path.join(paths.data_path, shared.opts.save_log_fn)
+            if not fn.endswith('.json'):
+                fn += '.json'
            entries = shared.readfile(fn)
            idx = len(list(entries))
            if idx == 0:
@@ -547,7 +549,7 @@ save_thread = threading.Thread(target=atomically_save_image, daemon=True)
 save_thread.start()


-def save_image(image, path, basename = '', seed=None, prompt=None, extension=shared.opts.samples_format, info=None, short_filename=False, no_prompt=False, grid=False, pnginfo_section_name='parameters', p=None, existing_info=None, forced_filename=None, suffix="", save_to_dirs=None): # pylint: disable=unused-argument
+def save_image(image, path, basename='', seed=None, prompt=None, extension=shared.opts.samples_format, info=None, short_filename=False, no_prompt=False, grid=False, pnginfo_section_name='parameters', p=None, existing_info=None, forced_filename=None, suffix='', save_to_dirs=None): # pylint: disable=unused-argument
    if image is None:
        shared.log.warning('Image is none')
        return None, None
@@ -556,26 +558,30 @@ def save_image(image, path, basename = '', seed=None, prompt=None, extension=sha
    if path is None or len(path) == 0: # set default path to avoid errors when functions are triggered manually or via api and param is not set
        path = shared.opts.outdir_save
    namegen = FilenameGenerator(p, seed, prompt, image, grid=grid)
+    suffix = suffix if suffix is not None else ''
+    basename = basename if basename is not None else ''
    if shared.opts.save_to_dirs:
        dirname = namegen.apply(shared.opts.directories_filename_pattern or "[prompt_words]")
        path = os.path.join(path, dirname)
    if forced_filename is None:
-        if short_filename or seed is None:
-            file_decoration = ""
        if shared.opts.samples_filename_pattern and len(shared.opts.samples_filename_pattern) > 0:
            file_decoration = shared.opts.samples_filename_pattern
        else:
            file_decoration = "[seq]-[prompt_words]"
        file_decoration = namegen.apply(file_decoration)
-        file_decoration += suffix
-    filename = os.path.join(path, f"{file_decoration}.{extension}") if basename == '' else os.path.join(path, f"{basename}-{file_decoration}.{extension}")
+        file_decoration += suffix if suffix is not None else ''
+        filename = os.path.join(path, f"{file_decoration}.{extension}") if basename == '' else os.path.join(path, f"{basename}-{file_decoration}.{extension}")
+    else:
+        forced_filename += suffix if suffix is not None else ''
+        filename = os.path.join(path, f"{forced_filename}.{extension}") if basename == '' else os.path.join(path, f"{basename}-{forced_filename}.{extension}")
    pnginfo = existing_info or {}
    if info is not None:
        pnginfo[pnginfo_section_name] = info
    params = script_callbacks.ImageSaveParams(image, p, filename, pnginfo)
    params.filename = namegen.sanitize(filename)
    dirname = os.path.dirname(params.filename)
-    os.makedirs(dirname, exist_ok=True)
+    if dirname is not None and len(dirname) > 0:
+        os.makedirs(dirname, exist_ok=True)
    # sequence
    if shared.opts.save_images_add_number or '[seq]' in params.filename:
        if '[seq]' not in params.filename:
@@ -211,6 +211,11 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s
    p.extra_generation_params['Resize mode'] = resize_mode
    if mask:
        p.extra_generation_params["Mask blur"] = mask_blur
+        p.extra_generation_params["Mask alpha"] = mask_alpha
+        p.extra_generation_params["Mask invert"] = inpainting_mask_invert
+        p.extra_generation_params["Mask content"] = inpainting_fill
+        p.extra_generation_params["Mask area"] = inpaint_full_res
+        p.extra_generation_params["Mask padding"] = inpaint_full_res_padding
    p.is_batch = mode == 5
    if p.is_batch:
        process_batch(p, img2img_batch_files, img2img_batch_input_dir, img2img_batch_output_dir, img2img_batch_inpaint_mask_dir, args)
@@ -87,9 +87,10 @@ class InterrogateModels:
    def load_blip_model(self):
        self.create_fake_fairscale()
        import models.blip # pylint: disable=no-name-in-module
-
+        model_path = os.path.join(paths.models_path, "BLIP")
+        shared.log.debug(f'Loading interrogate model: type=BLIP folder={model_path}')
        files = modelloader.load_models(
-            model_path=os.path.join(paths.models_path, "BLIP"),
+            model_path=model_path,
            model_url='https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_caption_capfilt_large.pth',
            ext_filter=[".pth"],
            download_name='model_base_caption_capfilt_large.pth',
@@ -334,23 +334,23 @@ def load_reference(name: str):
        return True


-modelloader_directories = {}
+cache_folders = {}
 cache_last = 0
 cache_time = 1


-def directory_has_changed(dir:str, *, recursive:bool=True) -> bool: # pylint: disable=redefined-builtin
+def directory_updated(path:str, *, recursive:bool=True) -> bool: # pylint: disable=redefined-builtin
    try:
-        dir = os.path.abspath(dir)
-        if dir not in modelloader_directories:
+        path = os.path.abspath(path)
+        if path not in cache_folders:
            return True
        if cache_last > (time.time() - cache_time):
            return False
-        if not (os.path.exists(dir) and os.path.isdir(dir) and os.path.getmtime(dir) == modelloader_directories[dir][0]):
+        if not (os.path.exists(path) and os.path.isdir(path) and os.path.getmtime(path) == cache_folders[path][0]):
            return True
        if recursive:
-            for _dir in modelloader_directories:
-                if _dir.startswith(dir) and _dir != dir and not (os.path.exists(_dir) and os.path.isdir(_dir) and os.path.getmtime(_dir) == modelloader_directories[_dir][0]):
+            for folder in cache_folders:
+                if folder.startswith(path) and folder != path and not (os.path.exists(folder) and os.path.isdir(folder) and os.path.getmtime(folder) == cache_folders[folder][0]):
                    return True
    except Exception as e:
        shared.log.error(f"Filesystem Error: {e.__class__.__name__}({e})")
@@ -358,44 +358,48 @@ def directory_has_changed(dir:str, *, recursive:bool=True) -> bool: # pylint: di
    return False


-def directory_directories(dir:str, *, recursive:bool=True) -> dict[str,tuple[float,list[str]]]: # pylint: disable=redefined-builtin
-    dir = os.path.abspath(dir)
-    if directory_has_changed(dir, recursive=recursive):
-        for _dir in list(modelloader_directories):
-            if os.path.exists(_dir) or os.path.isdir(_dir):
+def directory_list(path:str, *, recursive:bool=True) -> dict[str,tuple[float,list[str]]]: # pylint: disable=redefined-builtin
+    path = os.path.abspath(path)
+    res = {}
+    if not os.path.exists(path):
+        return res
+    if directory_updated(path, recursive=recursive):
+        for folder in list(cache_folders):
+            del cache_folders[folder]
+            if os.path.exists(folder) or os.path.isdir(folder):
+                continue
+        for folder, files in walk(path, lambda e, path: shared.log.debug(f"FS walk error: {e} {path}")):
+            if not os.path.exists(folder):
                continue
-            del modelloader_directories[_dir]
-        for _dir, _files in walk(dir, lambda e, path: shared.log.debug(f"FS walk error: {e} {path}")):
            try:
-                mtime = os.path.getmtime(_dir)
-                if _dir not in modelloader_directories or mtime != modelloader_directories[_dir][0]:
-                    modelloader_directories[_dir] = (mtime, [os.path.join(_dir, fn) for fn in _files])
+                mtime = os.path.getmtime(folder)
+                if folder not in cache_folders or mtime != cache_folders[folder][0]:
+                    cache_folders[folder] = (mtime, [os.path.join(folder, fn) for fn in files])
            except Exception as e:
                shared.log.error(f"Filesystem Error: {e.__class__.__name__}({e})")
-                del modelloader_directories[_dir]
-    res = {}
-    for _dir in modelloader_directories:
-        if _dir == dir or (recursive and _dir.startswith(dir)):
-            res[_dir] = modelloader_directories[_dir]
+                del cache_folders[folder]
+    for folder in cache_folders:
+        if folder == path or (recursive and folder.startswith(path)):
+            res[folder] = cache_folders[folder]
            if not recursive:
                break
    return res


-def directory_mtime(dir:str, *, recursive:bool=True) -> float: # pylint: disable=redefined-builtin
-    return float(max(0, *[mtime for mtime, _ in directory_directories(dir, recursive=recursive).values()]))
+def directory_mtime(path:str, *, recursive:bool=True) -> float: # pylint: disable=redefined-builtin
+    return float(max(0, *[mtime for mtime, _ in directory_list(path, recursive=recursive).values()]))


 def directories_file_paths(directories:dict) -> list[str]:
    return sum([dat[1] for dat in directories.values()],[])


-def unique_directories(directories:list[str], *, recursive:bool=True) -> list[str]:
+def directories_unique(directories:list[str], *, recursive:bool=True) -> list[str]:
    '''Ensure no empty, or duplicates'''
-    directories = { os.path.abspath(dir): True for dir in directories if dir }.keys()
+    directories = { os.path.abspath(path): True for path in directories if path }.keys()
    if recursive:
        '''If we are going recursive, then directories that are children of other directories are redundant'''
-        directories = [dir for dir in directories if not any(_dir != dir and dir.startswith(os.path.join(_dir,'')) for _dir in directories)]
+        directories = [path for path in directories if not any(d != path and path.startswith(os.path.join(d,'')) for d in directories)]
    return directories


@@ -404,7 +408,7 @@ def unique_paths(paths:list[str]) -> list[str]:


 def directory_files(*directories:list[str], recursive:bool=True) -> list[str]:
-    return unique_paths(sum([[*directories_file_paths(directory_directories(dir, recursive=recursive))] for dir in unique_directories(directories, recursive=recursive)],[]))
+    return unique_paths(sum([[*directories_file_paths(directory_list(d, recursive=recursive))] for d in directories_unique(directories, recursive=recursive)],[]))


 def extension_filter(ext_filter=None, ext_blacklist=None):
@@ -485,7 +489,7 @@ def load_models(model_path: str, model_url: str = None, command_path: str = None
    @param ext_filter: An optional list of filename extensions to filter by
    @return: A list of paths containing the desired model(s)
    """
-    places = unique_directories([model_path, command_path])
+    places = directories_unique([model_path, command_path])
    output = []
    try:
        output:list = [*filter(extension_filter(ext_filter, ext_blacklist), directory_files(*places))]
@@ -7,6 +7,7 @@ import random
 import warnings
 from contextlib import nullcontext
 from typing import Any, Dict, List
+from dataclasses import dataclass, field
 import torch
 import numpy as np
 import cv2
@@ -115,6 +116,7 @@ def txt2img_image_conditioning(sd_model, x, width, height):
        return x.new_zeros(x.shape[0], 5, 1, 1, dtype=x.dtype, device=x.device)


+@dataclass(repr=False)
 class StableDiffusionProcessing:
    """
    The first set of paramaters: sd_models -> do_not_reload_embeddings represent the minimum required to create a StableDiffusionProcessing
@@ -164,11 +166,6 @@ class StableDiffusionProcessing:
        self.disable_extra_networks = False
        self.token_merging_ratio = 0
        self.token_merging_ratio_hr = 0
-        if not seed_enable_extras:
-            self.subseed = -1
-            self.subseed_strength = 0
-            self.seed_resize_from_h = 0
-            self.seed_resize_from_w = 0
        self.scripts = None
        self.script_args = script_args or []
        self.per_script_args = {}
@@ -203,11 +200,39 @@ class StableDiffusionProcessing:
        self.all_hr_negative_prompts = []
        self.comments = {}
        self.is_api = False
+        self.scripts_value: modules.scripts.ScriptRunner = field(default=None, init=False)
+        self.script_args_value: list = field(default=None, init=False)
+        self.scripts_setup_complete: bool = field(default=False, init=False)
+

    @property
    def sd_model(self):
        return shared.sd_model

+    @property
+    def scripts(self):
+        return self.scripts_value
+
+    @scripts.setter
+    def scripts(self, value):
+        self.scripts_value = value
+        if self.scripts_value and self.script_args_value and not self.scripts_setup_complete:
+            self.setup_scripts()
+
+    @property
+    def script_args(self):
+        return self.script_args_value
+
+    @script_args.setter
+    def script_args(self, value):
+        self.script_args_value = value
+        if self.scripts_value and self.script_args_value and not self.scripts_setup_complete:
+            self.setup_scripts()
+
+    def setup_scripts(self):
+        self.scripts_setup_complete = True
+        self.scripts.setup_scrips(self, is_ui=not self.is_api)
+
    def comment(self, text):
        self.comments[text] = 1

@@ -558,7 +583,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
        args["Denoising strength"] = p.denoising_strength
        args["Latent sampler"] = p.latent_sampler
        args["Image CFG scale"] = p.image_cfg_scale
-        args["CFG rescale"] = p.diffusers_guidance_rescale if shared.backend == shared.Backend.DIFFUSERS else None
+        args["CFG rescale"] = p.diffusers_guidance_rescale
    if 'refine' in p.ops:
        args["Second pass"] = p.enable_hr
        args["Refiner"] = None if (not shared.opts.add_model_name_to_info) or (not shared.sd_refiner) or (not shared.sd_refiner.sd_checkpoint_info.model_name) else shared.sd_refiner.sd_checkpoint_info.model_name.replace(',', '').replace(':', '')
@@ -567,12 +592,13 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
        args['Refiner start'] = p.refiner_start
        args["Hires steps"] = p.hr_second_pass_steps
        args["Latent sampler"] = p.latent_sampler
-        args["CFG rescale"] = p.diffusers_guidance_rescale if shared.backend == shared.Backend.DIFFUSERS else None
+        args["CFG rescale"] = p.diffusers_guidance_rescale
    if 'img2img' in p.ops or 'inpaint' in p.ops:
        args["Init image size"] = f"{getattr(p, 'init_img_width', 0)}x{getattr(p, 'init_img_height', 0)}"
        args["Init image hash"] = getattr(p, 'init_img_hash', None)
        args["Mask weight"] = getattr(p, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) if p.is_using_inpainting_conditioning else None
        args['Resize mode'] = getattr(p, 'resize_mode', None)
+        args['Resize scale'] = getattr(p, 'scale_by', None)
        args["Mask blur"] = p.mask_blur if getattr(p, 'mask', None) is not None and getattr(p, 'mask_blur', 0) > 0 else None
        args["Denoising strength"] = getattr(p, 'denoising_strength', None)
    if 'face' in p.ops:
@@ -651,7 +677,7 @@ def print_profile(profile, msg: str):
 def process_images(p: StableDiffusionProcessing) -> Processed:
    if not hasattr(p.sd_model, 'sd_checkpoint_info'):
        return None
-    if p.scripts is not None:
+    if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
        p.scripts.before_process(p)
    stored_opts = {}
    for k, v in p.override_settings.copy().items():
@@ -726,7 +752,13 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
    return res


-def validate_sample(sample):
+def validate_sample(tensor):
+    if tensor.dtype == torch.bfloat16: # numpy does not support bf16
+        tensor = tensor.to(torch.float16)
+    if shared.backend == shared.Backend.ORIGINAL:
+        sample = 255.0 * np.moveaxis(tensor.cpu().numpy(), 0, 2)
+    else:
+        sample = 255. * tensor
    with warnings.catch_warnings(record=True) as w:
        cast = sample.astype(np.uint8)
    if len(w) > 0:
@@ -771,7 +803,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
        p.all_subseeds = [int(subseed) + x for x in range(len(p.all_prompts))]
    if os.path.exists(shared.opts.embeddings_dir) and not p.do_not_reload_embeddings and shared.backend == shared.Backend.ORIGINAL:
        modules.sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings(force_reload=False)
-    if p.scripts is not None:
+    if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
        p.scripts.process(p)
    infotexts = []
    output_images = []
@@ -809,7 +841,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
            p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n + 1) * p.batch_size]
            p.seeds = p.all_seeds[n * p.batch_size:(n + 1) * p.batch_size]
            p.subseeds = p.all_subseeds[n * p.batch_size:(n + 1) * p.batch_size]
-            if p.scripts is not None:
+            if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
                p.scripts.before_process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds)
            if len(p.prompts) == 0:
                break
@@ -817,7 +849,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
            if not p.disable_extra_networks:
                with devices.autocast():
                    modules.extra_networks.activate(p, extra_network_data)
-            if p.scripts is not None:
+            if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
                p.scripts.process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds)
            if n == 0:
                with open(os.path.join(modules.paths.data_path, "params.txt"), "w", encoding="utf8") as file:
@@ -866,9 +898,9 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
            if shared.cmd_opts.lowvram or shared.cmd_opts.medvram and shared.backend == shared.Backend.ORIGINAL:
                modules.lowvram.send_everything_to_cpu()
                devices.torch_gc()
-            if p.scripts is not None:
+            if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
                p.scripts.postprocess_batch(p, x_samples_ddim, batch_number=n)
-            if p.scripts is not None:
+            if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
                p.prompts = p.all_prompts[n * p.batch_size:(n + 1) * p.batch_size]
                p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n + 1) * p.batch_size]
                batch_params = modules.scripts.PostprocessBatchListArgs(list(x_samples_ddim))
@@ -884,7 +916,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                    image = x_sample
                    x_sample = np.array(x_sample)
                else:
-                    x_sample = 255. * (np.moveaxis(x_sample.cpu().numpy(), 0, 2) if shared.backend == shared.Backend.ORIGINAL else x_sample)
                    x_sample = validate_sample(x_sample)
                    image = Image.fromarray(x_sample)
                if p.restore_faces:
@@ -897,7 +928,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                    p.ops.append('face')
                    x_sample = modules.face_restoration.restore_faces(x_sample)
                    image = Image.fromarray(x_sample)
-                if p.scripts is not None:
+                if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
                    pp = modules.scripts.PostprocessImageArgs(image)
                    p.scripts.postprocess_image(p, pp)
                    image = pp.image
@@ -962,7 +993,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
        index_of_first_image=index_of_first_image,
        infotexts=infotexts,
    )
-    if p.scripts is not None and not (shared.state.interrupted or shared.state.skipped):
+    if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner) and not (shared.state.interrupted or shared.state.skipped):
        p.scripts.postprocess(p, res)
    return res

@@ -1088,7 +1119,6 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
                decoded_samples = decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae), self.full_quality)
                decoded_samples = torch.clamp((decoded_samples + 1.0) / 2.0, min=0.0, max=1.0)
                for i, x_sample in enumerate(decoded_samples):
-                    x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2)
                    x_sample = validate_sample(x_sample)
                    image = Image.fromarray(x_sample)
                    bak_extra_generation_params, bak_restore_faces = self.extra_generation_params, self.restore_faces
@@ -1104,7 +1134,6 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
                    decoded_samples = torch.clamp((decoded_samples + 1.0) / 2.0, min=0.0, max=1.0)
                batch_images = []
                for _i, x_sample in enumerate(decoded_samples):
-                    x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2)
                    x_sample = validate_sample(x_sample)
                    image = Image.fromarray(x_sample)
                    image = images.resize_image(1, image, target_width, target_height, upscaler_name=self.hr_upscaler)
@@ -1185,7 +1214,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
    def init(self, all_prompts, all_seeds, all_subseeds):
        if shared.backend == shared.Backend.DIFFUSERS and self.image_mask is not None:
            shared.sd_model = modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.INPAINTING)
-            self.sd_model.dtype = self.sd_model.unet.dtype
+            # self.sd_model.dtype = self.sd_model.unet.dtype
        elif shared.backend == shared.Backend.DIFFUSERS and self.image_mask is None:
            shared.sd_model = modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.IMAGE_2_IMAGE)

@@ -89,12 +89,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro

    def full_vae_decode(latents, model):
        t0 = time.time()
-        if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False):
+        if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False) and hasattr(model, 'unet'):
            shared.log.debug('Moving to CPU: model=UNet')
            unet_device = model.unet.device
            model.unet.to(devices.cpu)
            devices.torch_gc()
-        if not shared.cmd_opts.lowvram and not shared.opts.diffusers_seq_cpu_offload:
+        if not shared.cmd_opts.lowvram and not shared.opts.diffusers_seq_cpu_offload and hasattr(model, 'vae'):
            model.vae.to(devices.device)
        latents.to(model.vae.device)

@@ -104,7 +104,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
            latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype)

        decoded = model.vae.decode(latents / model.vae.config.scaling_factor, return_dict=False)[0]
-        if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False):
+        if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False) and hasattr(model, 'unet'):
            model.unet.to(unet_device)
        t1 = time.time()
        shared.log.debug(f'VAE decode: name={sd_vae.loaded_vae_file if sd_vae.loaded_vae_file is not None else "baked"} dtype={model.vae.dtype} upcast={upcast} images={latents.shape[0]} latents={latents.shape} time={round(t1-t0, 3)}')
@@ -112,15 +112,15 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro

    def full_vae_encode(image, model):
        shared.log.debug(f'VAE encode: name={sd_vae.loaded_vae_file if sd_vae.loaded_vae_file is not None else "baked"} dtype={model.vae.dtype} upcast={model.vae.config.get("force_upcast", None)}')
-        if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False):
+        if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False) and hasattr(model, 'unet'):
            shared.log.debug('Moving to CPU: model=UNet')
            unet_device = model.unet.device
            model.unet.to(devices.cpu)
            devices.torch_gc()
-        if not shared.cmd_opts.lowvram and not shared.opts.diffusers_seq_cpu_offload:
+        if not shared.cmd_opts.lowvram and not shared.opts.diffusers_seq_cpu_offload and hasattr(model, 'vae'):
            model.vae.to(devices.device)
        encoded = model.vae.encode(image.to(model.vae.device, model.vae.dtype)).latent_dist.sample()
-        if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False):
+        if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False) and hasattr(model, 'unet'):
            model.unet.to(unet_device)
        return encoded

@@ -263,6 +263,11 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                    args['negative_pooled_prompt_embeds'] = negative_pooled
            else:
                args['negative_prompt'] = negative_prompts
+        if hasattr(model, 'scheduler') and hasattr(model.scheduler, 'noise_sampler_seed') and hasattr(model.scheduler, 'noise_sampler'):
+            model.scheduler.noise_sampler = None # noise needs to be reset instead of using cached values
+            model.scheduler.noise_sampler_seed = seeds[0] # some schedulers have internal noise generator and do not use pipeline generator
+        if 'noise_sampler_seed' in possible:
+            args['noise_sampler_seed'] = seeds[0]
        if 'guidance_scale' in possible:
            args['guidance_scale'] = p.cfg_scale
        if 'generator' in possible:
@@ -294,6 +299,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
        clean.pop('callback', None)
        clean.pop('callback_steps', None)
        clean.pop('callback_on_step_end', None)
+        clean.pop('callback_on_step_end_tensor_inputs', None)
        if 'latents' in clean:
            clean['latents'] = clean['latents'].shape
        if 'image' in clean:
@@ -371,29 +377,35 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
    if shared.opts.diffusers_move_base and not getattr(shared.sd_model, 'has_accelerate', False):
        shared.sd_model.to(devices.device)

-    is_img2img = bool(sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or
-                      sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING)
+    is_img2img = bool(sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING)
    use_refiner_start = bool(is_refiner_enabled and not p.is_hr_pass and not is_img2img and p.refiner_start > 0 and p.refiner_start < 1)
    use_denoise_start = bool(is_img2img and p.refiner_start > 0 and p.refiner_start < 1)

    def calculate_base_steps():
+        steps = p.steps
        if use_refiner_start:
-            return int(p.steps // p.refiner_start + 1) if shared.sd_model_type == 'sdxl' else p.steps
-        elif use_denoise_start and shared.sd_model_type == 'sdxl':
-            return int(p.steps // (1 - p.refiner_start))
-        elif is_img2img:
-            return int(p.steps // p.denoising_strength + 1)
-        else:
-            return p.steps
+            steps = (p.steps // (1.0 - p.refiner_start)) if shared.sd_model_type == 'sdxl' else p.steps
+        if os.environ.get('SD_STEPS_DEBUG', None) is not None:
+            shared.log.debug(f'Steps: type=base input={p.steps} output={steps} refiner={use_refiner_start}')
+        return max(2, int(steps))
+
+    def calculate_hires_steps():
+        # denoising strength is applied to steps by diffusers so this is no-op
+        # steps = (p.hr_second_pass_steps * p.denoising_strength) if p.hr_second_pass_steps > 0 else (p.steps * p.denoising_strength)
+        steps = p.hr_second_pass_steps if p.hr_second_pass_steps > 0 else p.steps
+        if os.environ.get('SD_STEPS_DEBUG', None) is not None:
+            shared.log.debug(f'Steps: type=hires input={p.hr_second_pass_steps} output={steps} denoise={p.denoising_strength}')
+        return max(2, int(steps))

    def calculate_refiner_steps():
-        refiner_is_sdxl = bool("StableDiffusionXL" in shared.sd_refiner.__class__.__name__)
-        if p.refiner_start > 0 and p.refiner_start < 1 and refiner_is_sdxl:
-            refiner_steps = int(p.refiner_steps // (1 - p.refiner_start))
+        # diffusers apply additional math to refiner steps, but we leave numbers as-is without correction
+        if p.refiner_start > 0 and p.refiner_start < 1:
+            steps = ((1 - p.refiner_start) * p.refiner_steps) if p.refiner_steps > 0 else ((1 - p.refiner_start) * p.steps)
        else:
-            refiner_steps = int(p.refiner_steps // p.denoising_strength + 1) if refiner_is_sdxl else p.refiner_steps
-        p.refiner_steps = min(99, refiner_steps)
-        return p.refiner_steps
+            steps = (p.denoising_strength * p.refiner_steps) if p.refiner_steps > 0 else (p.denoising_strength * p.steps)
+        if os.environ.get('SD_STEPS_DEBUG', None) is not None:
+            shared.log.debug(f'Steps: type=refiner input={p.refiner_steps} output={steps} start={p.refiner_start} denoise={p.denoising_strength}')
+        return max(2, int(steps))

    # pipeline type is set earlier in processing, but check for sanity
    if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE and len(getattr(p, 'init_images' ,[])) == 0: # reset pipeline
@@ -451,15 +463,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
                recompile_model(hires=True)
                update_sampler(shared.sd_model, second_pass=True)
-                if p.hr_second_pass_steps == 0:
-                    p.hr_second_pass_steps = p.steps
                hires_args = set_pipeline_args(
                    model=shared.sd_model,
                    prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts,
                    negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts,
                    prompts_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts,
                    negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts,
-                    num_inference_steps=int(p.hr_second_pass_steps // p.denoising_strength + 1),
+                    num_inference_steps=calculate_hires_steps(),
                    eta=shared.opts.scheduler_eta,
                    guidance_scale=p.image_cfg_scale if p.image_cfg_scale is not None else p.cfg_scale,
                    guidance_rescale=p.diffusers_guidance_rescale,
@@ -511,12 +521,11 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                image = vae_decode(latents=image, model=shared.sd_model, full_quality=p.full_quality, output_type='pil')
                p.extra_generation_params['Noise level'] = noise_level
                output_type = 'np'
-            calculate_refiner_steps()
            refiner_args = set_pipeline_args(
                model=shared.sd_refiner,
                prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts[i],
                negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts[i],
-                num_inference_steps=p.refiner_steps,
+                num_inference_steps=calculate_refiner_steps(),
                eta=shared.opts.scheduler_eta,
                # strength=p.denoising_strength,
                noise_level=noise_level, # StableDiffusionUpscalePipeline only
@@ -268,7 +268,7 @@ def select_checkpoint(op='model'):
        shared.log.info(f'Select: {op}="{checkpoint_info.title if checkpoint_info is not None else None}"')
        return checkpoint_info
    if len(checkpoints_list) == 0 and not shared.cmd_opts.no_download:
-        shared.log.error("Cannot generate without a checkpoint")
+        shared.log.warning("Cannot generate without a checkpoint")
        shared.log.info("Set system paths to use existing folders in a different location")
        shared.log.info("Or use --ckpt <path-to-checkpoint> to force using existing checkpoint")
        return None
@@ -646,6 +646,14 @@ def detect_pipeline(f: str, op: str = 'model'):
                guess = 'Stable Diffusion XL Instruct'
            else:
                guess = 'Stable Diffusion'
+            if 'LCM_' in f or 'LCM-' in f:
+                if shared.backend == shared.Backend.ORIGINAL:
+                    shared.log.warning(f'Model detected as LCM model, but attempting to load using backend=original: {op}={f} size={size} MB')
+                guess = 'Latent Consistency Model'
+            if 'PixArt' in f:
+                if shared.backend == shared.Backend.ORIGINAL:
+                    shared.log.warning(f'Model detected as PixArt Alpha model, but attempting to load using backend=original: {op}={f} size={size} MB')
+                guess = 'PixArt Alpha'
            pipeline = shared_items.get_pipelines().get(guess, None)
            shared.log.info(f'Autodetect: {op}="{guess}" class={pipeline.__name__} file="{f}" size={size}MB')
        except Exception as e:
@@ -777,7 +785,7 @@ def set_diffuser_options(sd_model, vae, op: str):
        shared.log.debug(f'Setting {op} VAE: name={sd_vae.loaded_vae_file} upcast={sd_model.vae.config.get("force_upcast", None)}')
    if shared.opts.cross_attention_optimization == "xFormers" and hasattr(sd_model, 'enable_xformers_memory_efficient_attention'):
        sd_model.enable_xformers_memory_efficient_attention()
-    if shared.opts.opt_channelslast:
+    if shared.opts.opt_channelslast and hasattr(sd_model, 'unet'):
        shared.log.debug(f'Setting {op}: enable channels last')
        sd_model.unet.to(memory_format=torch.channels_last)

@@ -53,14 +53,14 @@ def create_sampler(name, model):
        sampler.config = config
        sampler.initialize(p=None)
        sampler.name = name
-        shared.log.debug(f'Sampler: sampler={sampler.name} config={sampler.config.options}')
+        shared.log.debug(f'Sampler: sampler="{sampler.name}" config={sampler.config.options}')
        return sampler
    elif shared.backend == shared.Backend.DIFFUSERS:
        sampler = config.constructor(model)
        if not hasattr(model, 'scheduler_config'):
            model.scheduler_config = sampler.sampler.config.copy()
        model.scheduler = sampler.sampler
-        shared.log.debug(f'Sampler: sampler={sampler.name} config={sampler.config}')
+        shared.log.debug(f'Sampler: sampler="{sampler.name}" config={sampler.config}')
        return sampler.sampler
    else:
        return None
@@ -17,6 +17,7 @@ try:
        UniPCMultistepScheduler,
        LMSDiscreteScheduler,
        KDPM2AncestralDiscreteScheduler,
+        LCMScheduler,
    )
 except Exception as e:
    import diffusers
@@ -40,6 +41,7 @@ config = {
    'LMSD': { 'use_karras_sigmas': False, 'timestep_spacing': 'linspace', 'steps_offset': 0 },
    'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0 },
    'UniPC': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True },
+    'LCM': { 'num_train_timesteps': 1000, 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False },
 }

 samplers_data_diffusers = [
@@ -58,6 +60,7 @@ samplers_data_diffusers = [
    sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}),
    sd_samplers_common.SamplerData('Euler a', lambda model: DiffusionSampler('Euler a', EulerAncestralDiscreteScheduler, model), [], {}),
    sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}),
+    sd_samplers_common.SamplerData('LCM', lambda model: DiffusionSampler('LCM', LCMScheduler, model), [], {}),
 ]

 class DiffusionSampler:
@@ -70,8 +73,10 @@ class DiffusionSampler:
            return
        for key, value in config.get('All', {}).items(): # apply global defaults
            self.config[key] = value
+        # shared.log.debug(f'Sampler: name={name} type=all config={self.config}')
        for key, value in config.get(name, {}).items(): # apply diffusers per-scheduler defaults
            self.config[key] = value
+        # shared.log.debug(f'Sampler: name={name} type=scheduler config={self.config}')
        if hasattr(model.scheduler, 'scheduler_config'): # find model defaults
            orig_config = model.scheduler.scheduler_config
        else:
@@ -79,9 +84,11 @@ class DiffusionSampler:
        for key, value in orig_config.items(): # apply model defaults
            if key in self.config:
                self.config[key] = value
+        # shared.log.debug(f'Sampler: name={name} type=model config={self.config}')
        for key, value in kwargs.items(): # apply user args, if any
            if key in self.config:
                self.config[key] = value
+        # shared.log.debug(f'Sampler: name={name} type=user config={self.config}')
        # finally apply user preferences
        if shared.opts.schedulers_prediction_type != 'default':
            self.config['prediction_type'] = shared.opts.schedulers_prediction_type
@@ -27,7 +27,7 @@ def list_crossattention():
 def get_pipelines():
    import diffusers
    from installer import log
-    pipelines = {
+    pipelines = { # note: not all pipelines can be used manually as they require prior pipeline next to decoder pipeline
        'Autodetect': None,
        'Stable Diffusion': getattr(diffusers, 'StableDiffusionPipeline', None),
        'Stable Diffusion Img2Img': getattr(diffusers, 'StableDiffusionImg2ImgPipeline', None),
@@ -37,9 +37,15 @@ def get_pipelines():
        'Stable Diffusion XL Img2Img': getattr(diffusers, 'StableDiffusionXLImg2ImgPipeline', None),
        'Stable Diffusion XL Inpaint': getattr(diffusers, 'StableDiffusionXLInpaintPipeline', None),
        'Stable Diffusion XL Instruct': getattr(diffusers, 'StableDiffusionXLInstructPix2PixPipeline', None),
+        'Latent Consistency Model': getattr(diffusers, 'LatentConsistencyModelPipeline', None),
+        'PixArt Alpha': getattr(diffusers, 'PixArtAlphaPipeline', None),
+        'UniDiffuser': getattr(diffusers, 'UniDiffuserPipeline', None),
+        'Wuerstchen': getattr(diffusers, 'WuerstchenCombinedPipeline', None),
+        'Kandinsky 2.1': getattr(diffusers, 'KandinskyPipeline', None),
+        'Kandinsky 2.2': getattr(diffusers, 'KandinskyV22Pipeline', None),
+        'DeepFloyd IF': getattr(diffusers, 'IFPipeline', None),
        'Custom Diffusers Pipeline': getattr(diffusers, 'DiffusionPipeline', None),
-        # 'Test': getattr(diffusers, 'TestPipeline', None),
-        # 'Kandinsky V1', 'Kandinsky V2', 'DeepFloyd IF', 'Shap-E', 'Kandinsky V1 Img2Img', 'Kandinsky V2 Img2Img', 'DeepFloyd IF Img2Img', 'Shap-E Img2Img',
+        # Segmind SSD-1B, Segmind Tiny
    }
    for k, v in pipelines.items():
        if k != 'Autodetect' and v is None:
@@ -5,7 +5,6 @@ import os
 import csv
 import json
 from installer import log
-from modules import paths


 class Style():
@@ -66,6 +65,8 @@ def apply_styles_to_extra(p, style: Style):

 class StyleDatabase:
    def __init__(self, opts):
+        from modules import paths
+
        self.no_style = Style("None")
        self.styles = {}
        self.path = opts.styles_dir
@@ -135,18 +136,33 @@ class StyleDatabase:
        return found[0] if len(found) > 0 else self.no_style

    def get_style_prompts(self, styles):
+        if styles is None or not isinstance(styles, list):
+            log.error(f'Invalid styles: {styles}')
+            return []
        return [self.find_style(x).prompt for x in styles]

    def get_negative_style_prompts(self, styles):
+        if styles is None or not isinstance(styles, list):
+            log.error(f'Invalid styles: {styles}')
+            return []
        return [self.find_style(x).negative_prompt for x in styles]

    def apply_styles_to_prompt(self, prompt, styles):
+        if styles is None or not isinstance(styles, list):
+            log.error(f'Invalid styles: {styles}')
+            return prompt
        return apply_styles_to_prompt(prompt, [self.find_style(x).prompt for x in styles])

    def apply_negative_styles_to_prompt(self, prompt, styles):
+        if styles is None or not isinstance(styles, list):
+            log.error(f'Invalid styles: {styles}')
+            return prompt
        return apply_styles_to_prompt(prompt, [self.find_style(x).negative_prompt for x in styles])

    def apply_styles_to_extra(self, p):
+        if p.styles is None or not isinstance(p.styles, list):
+            log.error(f'Invalid styles: {p.styles}')
+            return
        for style in p.styles:
            s = self.find_style(style)
            apply_styles_to_extra(p, s)
@@ -172,19 +188,25 @@ class StyleDatabase:
                log.error(f'Failed to save style: name={name} file={path} error={e}')
        count = len(list(self.styles))
        if count > 0:
-            log.debug(f'Saved styles: {path} {count}')
+            log.debug(f'Saved styles: folder="{path}" items={count}')

    def load_csv(self, legacy_file):
        if not os.path.isfile(legacy_file):
            return
        with open(legacy_file, "r", encoding="utf-8-sig", newline='') as file:
            reader = csv.DictReader(file, skipinitialspace=True)
+            num = 0
            for row in reader:
                try:
-                    self.styles[row["name"]] = Style(row["name"], row["prompt"] if "prompt" in row else row["text"], row.get("negative_prompt", ""))
+                    name = row["name"]
+                    prompt = row["prompt"] if "prompt" in row else row["text"]
+                    negative = row.get("negative_prompt", "") if "negative_prompt" in row else row.get("negative", "")
+                    self.styles[name] = Style(name, desc=name, prompt=prompt, negative_prompt=negative, extra="")
+                    log.debug(f'Migrated style: {self.styles[name].__dict__}')
+                    num += 1
                except Exception:
-                    log.error(f'Styles error: file={legacy_file} row={row}')
-            log.debug(f'Load legacy styles: file={legacy_file} items={len(self.styles.keys())}')
+                    log.error(f'Styles error: file="{legacy_file}" row={row}')
+            log.info(f'Load legacy styles: file="{legacy_file}" loaded={num} created={len(list(self.styles))}')

    """
    def save_csv(self, path: str) -> None:
@@ -511,43 +511,48 @@ def create_ui(startup_timer = None):
            txt_prompt_img.change(fn=modules.images.image_data, inputs=[txt_prompt_img], outputs=[txt2img_prompt, txt_prompt_img])

            txt2img_paste_fields = [
+                # prompt
                (txt2img_prompt, "Prompt"),
                (txt2img_negative_prompt, "Negative prompt"),
-                # (txt2img_prompt_styles, "Styles"),
-                (steps, "Steps"),
-                (seed, "Seed"),
-                (sampler_index, "Sampler"),
-                (cfg_scale, "CFG scale"),
+                # main
                (width, "Size-1"),
                (height, "Size-2"),
+                # sampler
+                (sampler_index, "Sampler"),
+                (steps, "Steps"),
+                # batch
+                (batch_count, "Batch-1"),
+                (batch_size, "Batch-2"),
+                # seed
+                (seed, "Seed"),
                (subseed, "Variation seed"),
                (subseed_strength, "Variation strength"),
+                # advanced
+                (cfg_scale, "CFG scale"),
                (clip_skip, "Clip skip"),
-                (latent_index, "Latent sampler"),
-                (latent_index, "Secondary sampler"),
-                (denoising_strength, "Denoising strength"),
-                (refiner_steps, "Refiner steps"),
-                (refiner_start, "Refiner start"),
+                (image_cfg_scale, "Image CFG scale"),
+                (diffusers_guidance_rescale, "CFG rescale"),
                (full_quality, "Full quality"),
                (restore_faces, "Face restoration"),
-                (batch_size, "Batch size"),
-                (batch_count, "Batch count"),
-                (seed_resize_from_w, "Seed resize from-1"),
-                (seed_resize_from_h, "Seed resize from-2"),
+                (tiling, "Tiling"),
+                # second pass
                (enable_hr, "Second pass"),
-                (hr_force, "Hires force"),
-                (hr_scale, "Hires upscale"),
+                (latent_index, "Latent sampler"),
+                (denoising_strength, "Denoising strength"),
                (hr_upscaler, "Hires upscaler"),
+                (hr_force, "Hires force"),
                (hr_second_pass_steps, "Hires steps"),
+                (hr_scale, "Hires upscale"),
                (hr_resize_x, "Hires resize-1"),
                (hr_resize_y, "Hires resize-2"),
-                (diffusers_guidance_rescale, "CFG rescale"),
-                (image_cfg_scale, "Image CFG scale"),
-                (refiner_steps, "Refiner steps"),
+                # refiner
                (refiner_start, "Refiner start"),
-                (tiling, "Tiling"),
-                (refiner_negative, "Negative2"),
+                (refiner_steps, "Refiner steps"),
                (refiner_prompt, "Prompt2"),
+                (refiner_negative, "Negative2"),
+                # hidden
+                (seed_resize_from_w, "Seed resize from-1"),
+                (seed_resize_from_h, "Seed resize from-2"),
                *modules.scripts.scripts_txt2img.infotext_fields
            ]
            parameters_copypaste.add_paste_fields("txt2img", None, txt2img_paste_fields, override_settings)
@@ -838,46 +843,45 @@ def create_ui(startup_timer = None):

            ui_extra_networks.setup_ui(extra_networks_ui_img2img, img2img_gallery)
            img2img_paste_fields = [
+                # prompt
                (img2img_prompt, "Prompt"),
                (img2img_negative_prompt, "Negative prompt"),
-                # (img2img_prompt_styles, "Styles"),
-                (steps, "Steps"),
-                (seed, "Seed"),
+                # sampler
                (sampler_index, "Sampler"),
-                (cfg_scale, "CFG scale"),
+                (steps, "Steps"),
+                # resize
+                (resize_mode, "Resize mode"),
                (width, "Size-1"),
                (height, "Size-2"),
+                (scale_by, "Resize scale"),
+                # batch
+                (batch_count, "Batch-1"),
+                (batch_size, "Batch-2"),
+                # seed
+                (seed, "Seed"),
                (subseed, "Variation seed"),
                (subseed_strength, "Variation strength"),
-                (full_quality, "Full quality"),
-                (clip_skip, "Clip skip"),
-                (latent_index, "Latent sampler"),
-                (latent_index, "Secondary sampler"),
+                # denoise
                (denoising_strength, "Denoising strength"),
+                (refiner_start, "Refiner start"),
+                # advanced
+                (cfg_scale, "CFG scale"),
+                (image_cfg_scale, "Image CFG scale"),
+                (clip_skip, "Clip skip"),
+                (diffusers_guidance_rescale, "CFG rescale"),
+                (full_quality, "Full quality"),
                (restore_faces, "Face restoration"),
-                (batch_size, "Batch size"),
-                (batch_count, "Batch count"),
+                (tiling, "Tiling"),
+                # inpaint
+                (mask_blur, "Mask blur"),
+                (mask_alpha, "Mask alpha"),
+                (inpainting_mask_invert, "Mask invert"),
+                (inpainting_fill, "Masked content"),
+                (inpaint_full_res, "Mask area"),
+                (inpaint_full_res_padding, "Masked padding"),
+                # hidden
                (seed_resize_from_w, "Seed resize from-1"),
                (seed_resize_from_h, "Seed resize from-2"),
-                (resize_mode, "Resize mode"),
-                (image_cfg_scale, "Image CFG scale"),
-                (diffusers_guidance_rescale, "CFG rescale"),
-                (tiling, "Tiling"),
-                (mask_blur, "Mask blur"),
-                # TODO scale_by add to paste fields
-                (scale_by, "UNKNOWN"),
-                # from txt2img
-                (hr_force, "Hires force"),
-                (hr_scale, "Hires upscale"),
-                (hr_upscaler, "Hires upscaler"),
-                (hr_second_pass_steps, "Hires steps"),
-                (hr_second_pass_steps, "Hires steps"),
-                (hr_resize_x, "Hires resize-1"),
-                (hr_resize_y, "Hires resize-2"),
-                (refiner_steps, "Refiner steps"),
-                (refiner_start, "Refiner start"),
-                (refiner_prompt, "Prompt2"),
-                (refiner_negative, "Negative2"),
                *modules.scripts.scripts_img2img.infotext_fields
            ]
            parameters_copypaste.add_paste_fields("img2img", init_img, img2img_paste_fields, override_settings)
@@ -93,6 +93,7 @@ def save_files(js_data, images, html_info, index):
            self.index_of_first_image = getattr(self, 'index_of_first_image', 0)
            self.infotexts = getattr(self, 'infotexts', [html_info])
            self.infotext = self.infotexts[0] if len(self.infotexts) > 0 else html_info
+            self.outpath_grids = shared.opts.outdir_grids or shared.opts.outdir_txt2img_grids
    try:
        data = json.loads(js_data)
    except Exception:
@@ -319,6 +319,7 @@ def create_html(search_text, sort_column):
    for ext in sorted(extensions_list, key=sort_function, reverse=sort_reverse):
        installed = get_installed(ext)
        author = ''
+        updated = datetime.timestamp(datetime.now())
        try:
            if 'github' in ext['url']:
                author = ext['url'].split('/')[-2].split(':')[-1] if '/' in ext['url'] else ext['url'].split(':')[1].split('/')[0]
@@ -40,9 +40,11 @@ card_full = '''
 '''
 card_list = '''
    <div class='card card-list' onclick={card_click} title='{name}' data-tab='{tabname}' data-page='{page}' data-name='{name}' data-filename='{filename}' data-tags='{tags}' data-mtime='{mtime}' data-size='{size}' data-search='{search}'>
-        <span class='details' title="Get details" onclick="showCardDetails(event)">&#x1f6c8;</span>&nbsp;
-        <div class='name'>{title}</div>&nbsp;
-        <div class='tags tags-list'></div>
+        <div style='display: flex'>
+            <span class='details' title="Get details" onclick="showCardDetails(event)">&#x1f6c8;</span>&nbsp;
+            <div class='name'>{title}</div>&nbsp;
+            <div class='tags tags-list'></div>
+        </div>
    </div>
 '''

@@ -222,6 +224,8 @@ class ExtraNetworksPage:
            self.items = []
            shared.log.error(f'Extra networks error listing items: class={self.__class__.__name__} tab={tabname} {e}')
        for item in self.items:
+            if item is None:
+                continue
            self.metadata[item["name"]] = item.get("metadata", {})
        t1 = time.time()
        debug(f'EN create-items: page={self.name} items={len(self.items)} time={t1-t0:.2f}')
@@ -237,7 +241,7 @@ class ExtraNetworksPage:
            return f"<div id='{tabname}_{self_name_id}_subdirs' class='extra-network-subdirs'></div><div id='{tabname}_{self_name_id}_cards' class='extra-network-cards'>Extra network page not ready<br>Click refresh to try again</div>"
        subdirs = {}
        allowed_folders = [os.path.abspath(x) for x in self.allowed_directories_for_previews()]
-        for parentdir, dirs in {d: modelloader.directory_directories(d) for d in allowed_folders}.items():
+        for parentdir, dirs in {d: modelloader.directory_list(d) for d in allowed_folders}.items():
            for tgt in dirs.keys():
                if shared.backend == shared.Backend.DIFFUSERS:
                    if os.path.join(paths.models_path, 'Reference') in tgt:
@@ -265,7 +269,8 @@ class ExtraNetworksPage:
        self.create_items(tabname)
        self.create_xyz_grid()
        htmls = []
-        self.items.sort(key=lambda x: x["mtime"], reverse=True)
+        if len(self.items) > 0 and self.items[0].get('mtime', None) is not None:
+            self.items.sort(key=lambda x: x["mtime"], reverse=True)
        for item in self.items:
            htmls.append(self.create_html(item, tabname))
        self.html += ''.join(htmls)
@@ -318,7 +323,7 @@ class ExtraNetworksPage:
            return 'html/card-no-preview.png'
        if shared.opts.diffusers_dir in path:
            path = os.path.relpath(path, shared.opts.diffusers_dir)
-            ref = os.path.join(paths.models_path, 'Reference')
+            ref = os.path.join('models', 'Reference')
            fn = os.path.join(ref, path.replace('models--', '').replace('\\', '/').split('/')[0])
            files = listdir(ref)
        else:
@@ -539,9 +544,11 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
            refresh_time = time.time()
        threads = []
        for page in get_pages():
-            # page.create_items(ui.tabname)
-            threads.append(threading.Thread(target=page.create_items, args=[ui.tabname]))
-            threads[-1].start()
+            if os.environ.get('SD_EN_DEBUG', None) is not None:
+                threads.append(threading.Thread(target=page.create_items, args=[ui.tabname]))
+                threads[-1].start()
+            else:
+                page.create_items(ui.tabname)
        for thread in threads:
            thread.join()
        for page in get_pages():
@@ -1,9 +1,11 @@
+import os
 import html
 import json
-import os
-from modules import shared, ui_extra_networks, sd_models, paths
+import concurrent
+from modules import shared, ui_extra_networks, sd_models

-reference_dir = os.path.join(paths.models_path, 'Reference')
+
+reference_dir = os.path.join('models', 'Reference')

 class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage):
    def __init__(self):
@@ -12,7 +14,7 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage):
    def refresh(self):
        shared.refresh_checkpoints()

-    def list_reference(self):
+    def list_reference(self): # pylint: disable=inconsistent-return-statements
        if shared.backend != shared.Backend.DIFFUSERS:
            return []
        reference_models = shared.readfile(os.path.join('html', 'reference.json'))
@@ -36,31 +38,38 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage):
                "description": v.get('desc', ''),
            }

+    def create_item(self, name):
+        record = None
+        try:
+            checkpoint: sd_models.CheckpointInfo = sd_models.checkpoints_list.get(name)
+            exists = os.path.exists(checkpoint.filename)
+            record = {
+                "type": 'Model',
+                "name": checkpoint.name,
+                "title": checkpoint.title,
+                "filename": checkpoint.filename,
+                "hash": checkpoint.shorthash,
+                "search_term": self.search_terms_from_path(checkpoint.title),
+                "preview": self.find_preview(checkpoint.filename),
+                "local_preview": f"{os.path.splitext(checkpoint.filename)[0]}.{shared.opts.samples_format}",
+                "metadata": checkpoint.metadata,
+                "onclick": '"' + html.escape(f"""return selectCheckpoint({json.dumps(name)})""") + '"',
+                "mtime": os.path.getmtime(checkpoint.filename) if exists else 0,
+                "size": os.path.getsize(checkpoint.filename) if exists else 0,
+            }
+            record["info"] = self.find_info(checkpoint.filename)
+            record["description"] = self.find_description(checkpoint.filename, record["info"])
+        except Exception as e:
+            shared.log.debug(f"Extra networks error: type=model file={name} {e}")
+        return record
+
    def list_items(self):
-        checkpoint: sd_models.CheckpointInfo
-        checkpoints = sd_models.checkpoints_list.copy()
-        for name, checkpoint in checkpoints.items():
-            try:
-                exists = os.path.exists(checkpoint.filename)
-                record = {
-                    "type": 'Model',
-                    "name": checkpoint.name,
-                    "title": checkpoint.title,
-                    "filename": checkpoint.filename,
-                    "hash": checkpoint.shorthash,
-                    "search_term": self.search_terms_from_path(checkpoint.title),
-                    "preview": self.find_preview(checkpoint.filename),
-                    "local_preview": f"{os.path.splitext(checkpoint.filename)[0]}.{shared.opts.samples_format}",
-                    "metadata": checkpoint.metadata,
-                    "onclick": '"' + html.escape(f"""return selectCheckpoint({json.dumps(name)})""") + '"',
-                    "mtime": os.path.getmtime(checkpoint.filename) if exists else 0,
-                    "size": os.path.getsize(checkpoint.filename) if exists else 0,
-                }
-                record["info"] = self.find_info(checkpoint.filename)
-                record["description"] = self.find_description(checkpoint.filename, record["info"])
-                yield record
-            except Exception as e:
-                shared.log.debug(f"Extra networks error: type=model file={name} {e}")
+        with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
+            future_items = {executor.submit(self.create_item, cp): cp for cp in list(sd_models.checkpoints_list.copy())}
+            for future in concurrent.futures.as_completed(future_items):
+                item = future.result()
+                if item is not None:
+                    yield item
        for record in self.list_reference():
            yield record

@@ -13,7 +13,6 @@ class ExtraNetworksPageHypernetworks(ui_extra_networks.ExtraNetworksPage):
    def list_items(self):
        for name, path in shared.hypernetworks.items():
            try:
-                fn = os.path.splitext(path)[0]
                name = os.path.relpath(os.path.splitext(path)[0], shared.opts.hypernetwork_dir)
                yield {
                    "type": 'Hypernetwork',
@@ -1,6 +1,7 @@
 import os
 import html
 import json
+import concurrent
 from modules import shared, extra_networks, ui_extra_networks, styles


@@ -62,35 +63,44 @@ class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage):
        }
        return item

-    def list_items(self):
-        for k, style in shared.prompt_styles.styles.items():
-            try:
-                fn = os.path.splitext(getattr(style, 'filename', ''))[0]
-                name = getattr(style, 'name', '')
-                if name == '':
-                    continue
-                txt = f'Prompt: {getattr(style, "prompt", "")}'
-                if len(getattr(style, 'negative_prompt', '')) > 0:
-                    txt += f'\nNegative: {style.negative_prompt}'
-                yield {
-                    "type": 'Style',
-                    "name": name,
-                    "title": k,
-                    "filename": style.filename,
-                    "search_term": f'{txt} {self.search_terms_from_path(name)}',
-                    "preview": style.preview if getattr(style, 'preview', None) is not None and style.preview.startswith('data:') else self.find_preview(fn),
-                    "description": style.description if getattr(style, 'description', None) is not None and len(style.description) > 0 else txt,
-                    "prompt": getattr(style, 'prompt', ''),
-                    "negative": getattr(style, 'negative_prompt', ''),
-                    "extra": getattr(style, 'extra', ''),
-                    "local_preview": f"{fn}.{shared.opts.samples_format}",
-                    "onclick": '"' + html.escape(f"""return selectStyle({json.dumps(name)})""") + '"',
-                    "mtime": getattr(style, 'mtime', 0),
-                    "size": os.path.getsize(style.filename),
-                }
-            except Exception as e:
-                shared.log.debug(f"Extra networks error: type=style file={k} {e}")
+    def create_item(self, k):
+        item = None
+        try:
+            style = shared.prompt_styles.styles.get(k)
+            fn = os.path.splitext(getattr(style, 'filename', ''))[0]
+            name = getattr(style, 'name', '')
+            if name == '':
+                return item
+            txt = f'Prompt: {getattr(style, "prompt", "")}'
+            if len(getattr(style, 'negative_prompt', '')) > 0:
+                txt += f'\nNegative: {style.negative_prompt}'
+            item = {
+                "type": 'Style',
+                "name": name,
+                "title": k,
+                "filename": style.filename,
+                "search_term": f'{txt} {self.search_terms_from_path(name)}',
+                "preview": style.preview if getattr(style, 'preview', None) is not None and style.preview.startswith('data:') else self.find_preview(fn),
+                "description": style.description if getattr(style, 'description', None) is not None and len(style.description) > 0 else txt,
+                "prompt": getattr(style, 'prompt', ''),
+                "negative": getattr(style, 'negative_prompt', ''),
+                "extra": getattr(style, 'extra', ''),
+                "local_preview": f"{fn}.{shared.opts.samples_format}",
+                "onclick": '"' + html.escape(f"""return selectStyle({json.dumps(name)})""") + '"',
+                "mtime": getattr(style, 'mtime', 0),
+                "size": os.path.getsize(style.filename),
+            }
+        except Exception as e:
+            shared.log.debug(f"Extra networks error: type=style file={k} {e}")
+        return item

+    def list_items(self):
+        with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
+            future_items = {executor.submit(self.create_item, style): style for style in list(shared.prompt_styles.styles)}
+            for future in concurrent.futures.as_completed(future_items):
+                item = future.result()
+                if item is not None:
+                    yield item

    def allowed_directories_for_previews(self):
        return [v for v in [shared.opts.styles_dir] if v is not None] + ['html']
@@ -1,5 +1,6 @@
 import json
 import os
+import concurrent
 from modules import shared, sd_hijack, sd_models, ui_extra_networks
 from modules.textual_inversion.textual_inversion import Embedding

@@ -8,6 +9,7 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage):
    def __init__(self):
        super().__init__('Embedding')
        self.allow_negative_prompt = True
+        self.embeddings = []

    def refresh(self):
        if sd_models.model_data.sd_model is None:
@@ -17,51 +19,61 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage):
        elif hasattr(sd_models.model_data.sd_model, 'embedding_db'):
            sd_models.model_data.sd_model.embedding_db.load_textual_inversion_embeddings(force_reload=True)

+    def create_item(self, embedding: Embedding):
+        record = None
+        try:
+            path, _ext = os.path.splitext(embedding.filename)
+            tags = {}
+            if embedding.tag is not None:
+                tags[embedding.tag]=1
+            name = os.path.splitext(embedding.basename)[0]
+            record = {
+                "type": 'Embedding',
+                "name": name,
+                "filename": embedding.filename,
+                "preview": self.find_preview(embedding.filename),
+                "search_term": self.search_terms_from_path(name),
+                "prompt": json.dumps(f" {os.path.splitext(embedding.name)[0]}"),
+                "local_preview": f"{path}.{shared.opts.samples_format}",
+                "tags": tags,
+                "mtime": os.path.getmtime(embedding.filename),
+                "size": os.path.getsize(embedding.filename),
+            }
+            record["info"] = self.find_info(embedding.filename)
+            record["description"] = self.find_description(embedding.filename, record["info"])
+        except Exception as e:
+            shared.log.debug(f"Extra networks error: type=embedding file={embedding.filename} {e}")
+        return record
+
    def list_items(self):
+
        def list_folder(folder):
            for filename in os.listdir(folder):
                fn = os.path.join(folder, filename)
                if os.path.isfile(fn) and (fn.lower().endswith(".pt") or fn.lower().endswith(".safetensors")):
                    embedding = Embedding(vec=0, name=os.path.basename(fn), filename=fn)
                    embedding.filename = fn
-                    embeddings.append(embedding)
+                    self.embeddings.append(embedding)
                elif os.path.isdir(fn) and not fn.startswith('.'):
                    list_folder(fn)

        if sd_models.model_data.sd_model is None:
-            embeddings = []
+            self.embeddings = []
            list_folder(shared.opts.embeddings_dir)
        elif shared.backend == shared.Backend.ORIGINAL:
-            embeddings = list(sd_hijack.model_hijack.embedding_db.word_embeddings.values())
+            self.embeddings = list(sd_hijack.model_hijack.embedding_db.word_embeddings.values())
        elif hasattr(sd_models.model_data.sd_model, 'embedding_db'):
-            embeddings = list(sd_models.model_data.sd_model.embedding_db.word_embeddings.values())
+            self.embeddings = list(sd_models.model_data.sd_model.embedding_db.word_embeddings.values())
        else:
-            embeddings = []
-        embeddings = sorted(embeddings, key=lambda emb: emb.filename)
-        for embedding in embeddings:
-            try:
-                path, _ext = os.path.splitext(embedding.filename)
-                tags = {}
-                if embedding.tag is not None:
-                    tags[embedding.tag]=1
-                name = os.path.splitext(embedding.basename)[0]
-                record = {
-                    "type": 'Embedding',
-                    "name": name,
-                    "filename": embedding.filename,
-                    "preview": self.find_preview(embedding.filename),
-                    "search_term": self.search_terms_from_path(name),
-                    "prompt": json.dumps(f" {os.path.splitext(embedding.name)[0]}"),
-                    "local_preview": f"{path}.{shared.opts.samples_format}",
-                    "tags": tags,
-                    "mtime": os.path.getmtime(embedding.filename),
-                    "size": os.path.getsize(embedding.filename),
-                }
-                record["info"] = self.find_info(embedding.filename)
-                record["description"] = self.find_description(embedding.filename, record["info"])
-                yield record
-            except Exception as e:
-                shared.log.debug(f"Extra networks error: type=embedding file={embedding.filename} {e}")
+            self.embeddings = []
+        self.embeddings = sorted(self.embeddings, key=lambda emb: emb.filename)
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
+            future_items = {executor.submit(self.create_item, net): net for net in self.embeddings}
+            for future in concurrent.futures.as_completed(future_items):
+                item = future.result()
+                if item is not None:
+                    yield item

    def allowed_directories_for_previews(self):
        return list(sd_hijack.model_hijack.embedding_db.embedding_dirs)
@@ -35,7 +35,7 @@ class BatchWriter:
 def load(clip_model_name):
    global ci # pylint: disable=global-statement
    if ci is None:
-        config = Config(device=devices.get_optimal_device(), cache_path=os.path.join(paths.models_path, 'clip-interrogator'), clip_model_name=clip_model_name, quiet=True)
+        config = Config(device=devices.get_optimal_device(), cache_path=os.path.join(paths.models_path, 'Interrogator'), clip_model_name=clip_model_name, quiet=True)
        if low_vram:
            config.apply_low_vram_defaults()
        shared.log.info(f'Interrogate load: config={config}')
@@ -51,6 +51,23 @@ class Upscaler:
        except Exception:
            pass

+    def find_folder(self, folder, scalers, loaded):
+        for fn in os.listdir(folder): # from folder
+            file_name = os.path.join(folder, fn)
+            if os.path.isdir(file_name):
+                self.find_folder(file_name, scalers, loaded)
+                continue
+            if not file_name.endswith('.pth') and not file_name.endswith('.pt'):
+                continue
+            if file_name not in loaded:
+                model_name = os.path.splitext(fn)[0]
+                scaler = UpscalerData(name=f'{self.name} {model_name}', path=file_name, upscaler=self)
+                scaler.custom = True
+                scalers.append(scaler)
+                loaded.append(file_name)
+                modules.shared.log.debug(f'Upscaler type={self.name} folder="{folder}" model="{model_name}" path="{file_name}"')
+                print(f'Upscaler type={self.name} folder="{folder}" model="{model_name}" path="{file_name}"')
+
    def find_scalers(self):
        scalers = []
        loaded = []
@@ -66,6 +83,8 @@ class Upscaler:
                # modules.shared.log.debug(f'Upscaler type={self.name} folder="{self.user_path}" model="{model[0]}" path="{model_path}"')
        if not os.path.exists(self.user_path):
            return scalers
+        self.find_folder(self.user_path, scalers, loaded)
+        """
        for fn in os.listdir(self.user_path): # from folder
            if not fn.endswith('.pth') and not fn.endswith('.pt'):
                continue
@@ -77,6 +96,7 @@ class Upscaler:
                scalers.append(scaler)
                loaded.append(file_name)
                # modules.shared.log.debug(f'Upscaler type={self.name} folder="{self.user_path}" model="{model_name}" path="{file_name}"')
+        """
        return scalers

    @abstractmethod
@@ -1,14 +1,16 @@
 import os
+from copy import deepcopy
 import torch
 from torch import nn
-from copy import deepcopy
-
 from facelib.utils import load_file_from_url
 from facelib.utils import download_pretrained_models
 from facelib.detection.yolov5face.models.common import Conv
-
 from .retinaface.retinaface import RetinaFace
 from .yolov5face.face_detector import YoloDetector
+from modules import paths
+
+
+model_dir = os.path.join(paths.models_path, 'Codeformer')


 def init_detection_model(model_name, half=False, device='cuda'):
@@ -32,7 +34,7 @@ def init_retinaface_model(model_name, half=False, device='cuda'):
    else:
        raise NotImplementedError(f'{model_name} is not implemented.')

-    model_path = load_file_from_url(url=model_url, model_dir='weights/facelib', progress=True, file_name=None)
+    model_path = load_file_from_url(url=model_url, model_dir=model_dir, progress=True, file_name=None)
    load_net = torch.load(model_path, map_location=lambda storage, loc: storage)
    # remove unnecessary 'module.'
    for k, v in deepcopy(load_net).items():
@@ -55,8 +57,8 @@ def init_yolov5face_model(model_name, device='cuda'):
        model_url = 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/yolov5n-face.pth'
    else:
        raise NotImplementedError(f'{model_name} is not implemented.')
-    
-    model_path = load_file_from_url(url=model_url, model_dir='weights/facelib', progress=True, file_name=None)
+
+    model_path = load_file_from_url(url=model_url, model_dir=model_dir, progress=True, file_name=None)
    load_net = torch.load(model_path, map_location=lambda storage, loc: storage)
    model.detector.load_state_dict(load_net, strict=True)
    model.detector.eval()
@@ -69,32 +71,3 @@ def init_yolov5face_model(model_name, device='cuda'):
            m._non_persistent_buffers_set = set()  # pytorch 1.6.0 compatibility

    return model
-
-
-# Download from Google Drive
-# def init_yolov5face_model(model_name, device='cuda'):
-#     if model_name == 'YOLOv5l':
-#         model = YoloDetector(config_name='facelib/detection/yolov5face/models/yolov5l.yaml', device=device)
-#         f_id = {'yolov5l-face.pth': '131578zMA6B2x8VQHyHfa6GEPtulMCNzV'}
-#     elif model_name == 'YOLOv5n':
-#         model = YoloDetector(config_name='facelib/detection/yolov5face/models/yolov5n.yaml', device=device)
-#         f_id = {'yolov5n-face.pth': '1fhcpFvWZqghpGXjYPIne2sw1Fy4yhw6o'}
-#     else:
-#         raise NotImplementedError(f'{model_name} is not implemented.')
-
-#     model_path = os.path.join('weights/facelib', list(f_id.keys())[0])
-#     if not os.path.exists(model_path):
-#         download_pretrained_models(file_ids=f_id, save_path_root='weights/facelib')
-
-#     load_net = torch.load(model_path, map_location=lambda storage, loc: storage)
-#     model.detector.load_state_dict(load_net, strict=True)
-#     model.detector.eval()
-#     model.detector = model.detector.to(device).float()
-
-#     for m in model.detector.modules():
-#         if type(m) in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]:
-#             m.inplace = True  # pytorch 1.7.0 compatibility
-#         elif isinstance(m, Conv):
-#             m._non_persistent_buffers_set = set()  # pytorch 1.6.0 compatibility
-
-#     return model
@@ -1,40 +1,26 @@
-import argparse
 import os
-from os import path as osp
-
+from modules import paths
 from basicsr.utils.download_util import load_file_from_url


-def download_pretrained_models(method, file_urls):
-    save_path_root = f'./weights/{method}'
-    os.makedirs(save_path_root, exist_ok=True)
+urls = {
+    'CodeFormer': {
+        'codeformer.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth'
+    },
+    'facelib': {
+        # 'yolov5l-face.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/yolov5l-face.pth',
+        'detection_Resnet50_Final.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/detection_Resnet50_Final.pth',
+        'parsing_parsenet.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/parsing_parsenet.pth'
+    }
+}

+
+def download_pretrained_models(file_urls):
+    model_dir = os.path.join(paths.models_path, 'Codeformer')
    for file_name, file_url in file_urls.items():
-        save_path = load_file_from_url(url=file_url, model_dir=save_path_root, progress=True, file_name=file_name)
+        load_file_from_url(url=file_url, model_dir=model_dir, progress=True, file_name=file_name)


 if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        'method',
-        type=str,
-        help=("Options: 'CodeFormer' 'facelib'. Set to 'all' to download all the models."))
-    args = parser.parse_args()
-
-    file_urls = {
-        'CodeFormer': {
-            'codeformer.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth'
-        },
-        'facelib': {
-            # 'yolov5l-face.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/yolov5l-face.pth',
-            'detection_Resnet50_Final.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/detection_Resnet50_Final.pth',
-            'parsing_parsenet.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/parsing_parsenet.pth'
-        }
-    }
-
-    if args.method == 'all':
-        for method in file_urls.keys():
-            download_pretrained_models(method, file_urls[method])
-    else:
-        download_pretrained_models(args.method, file_urls[args.method])
+    for method in urls.keys():
+        download_pretrained_models(urls[method])
@@ -1,18 +1,16 @@
-import argparse
 import os
+from modules import paths
 from os import path as osp
-
-# from basicsr.utils.download_util import download_file_from_google_drive
 import gdown


-def download_pretrained_models(method, file_ids):
-    save_path_root = f'./weights/{method}'
-    os.makedirs(save_path_root, exist_ok=True)
+model_dir = os.path.join(paths.models_path, 'Codeformer')

+
+def download_pretrained_models(file_ids):
    for file_name, file_id in file_ids.items():
        file_url = 'https://drive.google.com/uc?id='+file_id
-        save_path = osp.abspath(osp.join(save_path_root, file_name))
+        save_path = osp.abspath(osp.join(model_dir, file_name))
        if osp.exists(save_path):
            user_response = input(f'{file_name} already exist. Do you want to cover it? Y/N\n')
            if user_response.lower() == 'y':
@@ -29,21 +27,13 @@ def download_pretrained_models(method, file_ids):
            # download_file_from_google_drive(file_id, save_path)

 if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        'method',
-        type=str,
-        help=("Options: 'CodeFormer' 'facelib'. Set to 'all' to download all the models."))
-    args = parser.parse_args()
-
    # file name: file id
    # 'dlib': {
    #     'mmod_human_face_detector-4cb19393.dat': '1qD-OqY8M6j4PWUP_FtqfwUPFPRMu6ubX',
    #     'shape_predictor_5_face_landmarks-c4b1e980.dat': '1vF3WBUApw4662v9Pw6wke3uk1qxnmLdg',
    #     'shape_predictor_68_face_landmarks-fbdc2cb8.dat': '1tJyIVdCHaU6IDMDx86BZCxLGZfsWB8yq'
    # }
-    file_ids = {
+    urls = {
        'CodeFormer': {
            'codeformer.pth': '1v_E_vZvP-dQPF55Kc5SRCjaKTQXDz-JB'
        },
@@ -52,9 +42,5 @@ if __name__ == '__main__':
            'parsing_parsenet.pth': '16pkohyZZ8ViHGBk3QtVqxLZKzdo466bK'
        }
    }
-
-    if args.method == 'all':
-        for method in file_ids.keys():
-            download_pretrained_models(method, file_ids[method])
-    else:
-        download_pretrained_models(args.method, file_ids[args.method])
+    for method in urls.keys():
+        download_pretrained_models(urls[method])
@@ -1,280 +0,0 @@
-"""
-This file is used for deploying hugging face demo:
-https://huggingface.co/spaces/sczhou/CodeFormer
-"""
-
-import sys
-sys.path.append('CodeFormer')
-import os
-import cv2
-import torch
-import torch.nn.functional as F
-import gradio as gr
-
-from torchvision.transforms.functional import normalize
-
-from basicsr.utils import imwrite, img2tensor, tensor2img
-from basicsr.utils.download_util import load_file_from_url
-from facelib.utils.face_restoration_helper import FaceRestoreHelper
-from facelib.utils.misc import is_gray
-from basicsr.archs.rrdbnet_arch import RRDBNet
-from basicsr.utils.realesrgan_utils import RealESRGANer
-
-from basicsr.utils.registry import ARCH_REGISTRY
-
-
-os.system("pip freeze")
-
-pretrain_model_url = {
-    'codeformer': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth',
-    'detection': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/detection_Resnet50_Final.pth',
-    'parsing': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/parsing_parsenet.pth',
-    'realesrgan': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/RealESRGAN_x2plus.pth'
-}
-# download weights
-if not os.path.exists('CodeFormer/weights/CodeFormer/codeformer.pth'):
-    load_file_from_url(url=pretrain_model_url['codeformer'], model_dir='CodeFormer/weights/CodeFormer', progress=True, file_name=None)
-if not os.path.exists('CodeFormer/weights/facelib/detection_Resnet50_Final.pth'):
-    load_file_from_url(url=pretrain_model_url['detection'], model_dir='CodeFormer/weights/facelib', progress=True, file_name=None)
-if not os.path.exists('CodeFormer/weights/facelib/parsing_parsenet.pth'):
-    load_file_from_url(url=pretrain_model_url['parsing'], model_dir='CodeFormer/weights/facelib', progress=True, file_name=None)
-if not os.path.exists('CodeFormer/weights/realesrgan/RealESRGAN_x2plus.pth'):
-    load_file_from_url(url=pretrain_model_url['realesrgan'], model_dir='CodeFormer/weights/realesrgan', progress=True, file_name=None)
-
-# download images
-torch.hub.download_url_to_file(
-    'https://replicate.com/api/models/sczhou/codeformer/files/fa3fe3d1-76b0-4ca8-ac0d-0a925cb0ff54/06.png',
-    '01.png')
-torch.hub.download_url_to_file(
-    'https://replicate.com/api/models/sczhou/codeformer/files/a1daba8e-af14-4b00-86a4-69cec9619b53/04.jpg',
-    '02.jpg')
-torch.hub.download_url_to_file(
-    'https://replicate.com/api/models/sczhou/codeformer/files/542d64f9-1712-4de7-85f7-3863009a7c3d/03.jpg',
-    '03.jpg')
-torch.hub.download_url_to_file(
-    'https://replicate.com/api/models/sczhou/codeformer/files/a11098b0-a18a-4c02-a19a-9a7045d68426/010.jpg',
-    '04.jpg')
-torch.hub.download_url_to_file(
-    'https://replicate.com/api/models/sczhou/codeformer/files/7cf19c2c-e0cf-4712-9af8-cf5bdbb8d0ee/012.jpg',
-    '05.jpg')
-
-def imread(img_path):
-    img = cv2.imread(img_path)
-    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
-    return img
-
-# set enhancer with RealESRGAN
-def set_realesrgan():
-    half = True if torch.cuda.is_available() else False
-    model = RRDBNet(
-        num_in_ch=3,
-        num_out_ch=3,
-        num_feat=64,
-        num_block=23,
-        num_grow_ch=32,
-        scale=2,
-    )
-    upsampler = RealESRGANer(
-        scale=2,
-        model_path="CodeFormer/weights/realesrgan/RealESRGAN_x2plus.pth",
-        model=model,
-        tile=400,
-        tile_pad=40,
-        pre_pad=0,
-        half=half,
-    )
-    return upsampler
-
-upsampler = set_realesrgan()
-device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-codeformer_net = ARCH_REGISTRY.get("CodeFormer")(
-    dim_embd=512,
-    codebook_size=1024,
-    n_head=8,
-    n_layers=9,
-    connect_list=["32", "64", "128", "256"],
-).to(device)
-ckpt_path = "CodeFormer/weights/CodeFormer/codeformer.pth"
-checkpoint = torch.load(ckpt_path)["params_ema"]
-codeformer_net.load_state_dict(checkpoint)
-codeformer_net.eval()
-
-os.makedirs('output', exist_ok=True)
-
-def inference(image, background_enhance, face_upsample, upscale, codeformer_fidelity):
-    """Run a single prediction on the model"""
-    try: # global try
-        # take the default setting for the demo
-        has_aligned = False
-        only_center_face = False
-        draw_box = False
-        detection_model = "retinaface_resnet50"
-        print('Inp:', image, background_enhance, face_upsample, upscale, codeformer_fidelity)
-
-        img = cv2.imread(str(image), cv2.IMREAD_COLOR)
-        print('\timage size:', img.shape)
-
-        upscale = int(upscale) # convert type to int
-        if upscale > 4: # avoid memory exceeded due to too large upscale
-            upscale = 4 
-        if upscale > 2 and max(img.shape[:2])>1000: # avoid memory exceeded due to too large img resolution
-            upscale = 2 
-        if max(img.shape[:2]) > 1500: # avoid memory exceeded due to too large img resolution
-            upscale = 1
-            background_enhance = False
-            face_upsample = False
-
-        face_helper = FaceRestoreHelper(
-            upscale,
-            face_size=512,
-            crop_ratio=(1, 1),
-            det_model=detection_model,
-            save_ext="png",
-            use_parse=True,
-            device=device,
-        )
-        bg_upsampler = upsampler if background_enhance else None
-        face_upsampler = upsampler if face_upsample else None
-
-        if has_aligned:
-            # the input faces are already cropped and aligned
-            img = cv2.resize(img, (512, 512), interpolation=cv2.INTER_LINEAR)
-            face_helper.is_gray = is_gray(img, threshold=5)
-            if face_helper.is_gray:
-                print('\tgrayscale input: True')
-            face_helper.cropped_faces = [img]
-        else:
-            face_helper.read_image(img)
-            # get face landmarks for each face
-            num_det_faces = face_helper.get_face_landmarks_5(
-            only_center_face=only_center_face, resize=640, eye_dist_threshold=5
-            )
-            print(f'\tdetect {num_det_faces} faces')
-            # align and warp each face
-            face_helper.align_warp_face()
-
-        # face restoration for each cropped face
-        for idx, cropped_face in enumerate(face_helper.cropped_faces):
-            # prepare data
-            cropped_face_t = img2tensor(
-                cropped_face / 255.0, bgr2rgb=True, float32=True
-            )
-            normalize(cropped_face_t, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5), inplace=True)
-            cropped_face_t = cropped_face_t.unsqueeze(0).to(device)
-
-            try:
-                with torch.no_grad():
-                    output = codeformer_net(
-                        cropped_face_t, w=codeformer_fidelity, adain=True
-                    )[0]
-                    restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1))
-                del output
-                torch.cuda.empty_cache()
-            except RuntimeError as error:
-                print(f"Failed inference for CodeFormer: {error}")
-                restored_face = tensor2img(
-                    cropped_face_t, rgb2bgr=True, min_max=(-1, 1)
-                )
-
-            restored_face = restored_face.astype("uint8")
-            face_helper.add_restored_face(restored_face)
-
-        # paste_back
-        if not has_aligned:
-            # upsample the background
-            if bg_upsampler is not None:
-                # Now only support RealESRGAN for upsampling background
-                bg_img = bg_upsampler.enhance(img, outscale=upscale)[0]
-            else:
-                bg_img = None
-            face_helper.get_inverse_affine(None)
-            # paste each restored face to the input image
-            if face_upsample and face_upsampler is not None:
-                restored_img = face_helper.paste_faces_to_input_image(
-                    upsample_img=bg_img,
-                    draw_box=draw_box,
-                    face_upsampler=face_upsampler,
-                )
-            else:
-                restored_img = face_helper.paste_faces_to_input_image(
-                    upsample_img=bg_img, draw_box=draw_box
-                )
-
-        # save restored img
-        save_path = f'output/out.png'
-        imwrite(restored_img, str(save_path))
-
-        restored_img = cv2.cvtColor(restored_img, cv2.COLOR_BGR2RGB)
-        return restored_img, save_path
-    except Exception as error:
-        print('Global exception', error)
-        return None, None
-
-
-title = "CodeFormer: Robust Face Restoration and Enhancement Network"
-description = r"""<center><img src='https://user-images.githubusercontent.com/14334509/189166076-94bb2cac-4f4e-40fb-a69f-66709e3d98f5.png' alt='CodeFormer logo'></center>
-<b>Official Gradio demo</b> for <a href='https://github.com/sczhou/CodeFormer' target='_blank'><b>Towards Robust Blind Face Restoration with Codebook Lookup Transformer (NeurIPS 2022)</b></a>.<br>
-🔥 CodeFormer is a robust face restoration algorithm for old photos or AI-generated faces.<br>
-🤗 Try CodeFormer for improved stable-diffusion generation!<br>
-"""
-article = r"""
-If CodeFormer is helpful, please help to ⭐ the <a href='https://github.com/sczhou/CodeFormer' target='_blank'>Github Repo</a>. Thanks! 
-[![GitHub Stars](https://img.shields.io/github/stars/sczhou/CodeFormer?style=social)](https://github.com/sczhou/CodeFormer)
-
---
-
-📝 **Citation**
-
-If our work is useful for your research, please consider citing:
-```bibtex
-@inproceedings{zhou2022codeformer,
-    author = {Zhou, Shangchen and Chan, Kelvin C.K. and Li, Chongyi and Loy, Chen Change},
-    title = {Towards Robust Blind Face Restoration with Codebook Lookup TransFormer},
-    booktitle = {NeurIPS},
-    year = {2022}
-}
-```
-
-📋 **License**
-
-This project is licensed under <a rel="license" href="https://github.com/sczhou/CodeFormer/blob/master/LICENSE">S-Lab License 1.0</a>. 
-Redistribution and use for non-commercial purposes should follow this license.
-
-📧 **Contact**
-
-If you have any questions, please feel free to reach me out at <b>shangchenzhou@gmail.com</b>.
-
-<div>
-    🤗 Find Me:
-    <a href="https://twitter.com/ShangchenZhou"><img style="margin-top:0.5em; margin-bottom:0.5em" src="https://img.shields.io/twitter/follow/ShangchenZhou?label=%40ShangchenZhou&style=social" alt="Twitter Follow"></a> 
-    <a href="https://github.com/sczhou"><img style="margin-top:0.5em; margin-bottom:2em" src="https://img.shields.io/github/followers/sczhou?style=social" alt="Github Follow"></a>
-</div>
-
-<center><img src='https://visitor-badge-sczhou.glitch.me/badge?page_id=sczhou/CodeFormer' alt='visitors'></center>
-"""
-
-demo = gr.Interface(
-    inference, [
-        gr.inputs.Image(type="filepath", label="Input"),
-        gr.inputs.Checkbox(default=True, label="Background_Enhance"),
-        gr.inputs.Checkbox(default=True, label="Face_Upsample"),
-        gr.inputs.Number(default=2, label="Rescaling_Factor (up to 4)"),
-        gr.Slider(0, 1, value=0.5, step=0.01, label='Codeformer_Fidelity (0 for better quality, 1 for better identity)')
-    ], [
-        gr.outputs.Image(type="numpy", label="Output"),
-        gr.outputs.File(label="Download the output")
-    ],
-    title=title,
-    description=description,
-    article=article,       
-    examples=[
-        ['01.png', True, True, 2, 0.7],
-        ['02.jpg', True, True, 2, 0.7],
-        ['03.jpg', True, True, 2, 0.7],
-        ['04.jpg', True, True, 2, 0.1],
-        ['05.jpg', True, True, 2, 0.1]
-      ]
-    )
-
-demo.queue(concurrency_count=2)
-demo.launch()
@@ -1,30 +0,0 @@
-"""
-This file is used for deploying replicate demo:
-https://replicate.com/sczhou/codeformer
-"""
-
-build:
-  gpu: true
-  cuda: "11.3"
-  python_version: "3.8"
-  system_packages:
-    - "libgl1-mesa-glx"
-    - "libglib2.0-0"
-  python_packages:
-    - "ipython==8.4.0"
-    - "future==0.18.2"
-    - "lmdb==1.3.0"
-    - "scikit-image==0.19.3"
-    - "torch==1.11.0 --extra-index-url=https://download.pytorch.org/whl/cu113"
-    - "torchvision==0.12.0 --extra-index-url=https://download.pytorch.org/whl/cu113"
-    - "scipy==1.9.0"
-    - "gdown==4.5.1"
-    - "pyyaml==6.0"
-    - "tb-nightly==2.11.0a20220906"
-    - "tqdm==4.64.1"
-    - "yapf==0.32.0"
-    - "lpips==0.1.4"
-    - "Pillow==9.2.0"
-    - "opencv-python==4.6.0.66"
-
-predict: "predict.py:Predictor"
@@ -1,189 +0,0 @@
-"""
-This file is used for deploying replicate demo:
-https://replicate.com/sczhou/codeformer
-running: cog predict -i image=@inputs/whole_imgs/04.jpg -i codeformer_fidelity=0.5 -i upscale=2
-push: cog push r8.im/sczhou/codeformer
-"""
-
-import tempfile
-import cv2
-import torch
-from torchvision.transforms.functional import normalize
-try:
-    from cog import BasePredictor, Input, Path
-except Exception:
-    print('please install cog package')
-
-from basicsr.utils import imwrite, img2tensor, tensor2img
-from basicsr.archs.rrdbnet_arch import RRDBNet
-from basicsr.utils.realesrgan_utils import RealESRGANer
-from basicsr.utils.registry import ARCH_REGISTRY
-from facelib.utils.face_restoration_helper import FaceRestoreHelper
-
-
-class Predictor(BasePredictor):
-    def setup(self):
-        """Load the model into memory to make running multiple predictions efficient"""
-        self.device = "cuda:0"
-        self.upsampler = set_realesrgan()
-        self.net = ARCH_REGISTRY.get("CodeFormer")(
-            dim_embd=512,
-            codebook_size=1024,
-            n_head=8,
-            n_layers=9,
-            connect_list=["32", "64", "128", "256"],
-        ).to(self.device)
-        ckpt_path = "weights/CodeFormer/codeformer.pth"
-        checkpoint = torch.load(ckpt_path)[
-            "params_ema"
-        ]  # update file permission if cannot load
-        self.net.load_state_dict(checkpoint)
-        self.net.eval()
-
-    def predict(
-        self,
-        image: Path = Input(description="Input image"),
-        codeformer_fidelity: float = Input(
-            default=0.5,
-            ge=0,
-            le=1,
-            description="Balance the quality (lower number) and fidelity (higher number).",
-        ),
-        background_enhance: bool = Input(
-            description="Enhance background image with Real-ESRGAN", default=True
-        ),
-        face_upsample: bool = Input(
-            description="Upsample restored faces for high-resolution AI-created images",
-            default=True,
-        ),
-        upscale: int = Input(
-            description="The final upsampling scale of the image",
-            default=2,
-        ),
-    ) -> Path:
-        """Run a single prediction on the model"""
-
-        # take the default setting for the demo
-        has_aligned = False
-        only_center_face = False
-        draw_box = False
-        detection_model = "retinaface_resnet50"
-
-        self.face_helper = FaceRestoreHelper(
-            upscale,
-            face_size=512,
-            crop_ratio=(1, 1),
-            det_model=detection_model,
-            save_ext="png",
-            use_parse=True,
-            device=self.device,
-        )
-
-        bg_upsampler = self.upsampler if background_enhance else None
-        face_upsampler = self.upsampler if face_upsample else None
-
-        img = cv2.imread(str(image), cv2.IMREAD_COLOR)
-
-        if has_aligned:
-            # the input faces are already cropped and aligned
-            img = cv2.resize(img, (512, 512), interpolation=cv2.INTER_LINEAR)
-            self.face_helper.cropped_faces = [img]
-        else:
-            self.face_helper.read_image(img)
-            # get face landmarks for each face
-            num_det_faces = self.face_helper.get_face_landmarks_5(
-                only_center_face=only_center_face, resize=640, eye_dist_threshold=5
-            )
-            print(f"\tdetect {num_det_faces} faces")
-            # align and warp each face
-            self.face_helper.align_warp_face()
-
-        # face restoration for each cropped face
-        for idx, cropped_face in enumerate(self.face_helper.cropped_faces):
-            # prepare data
-            cropped_face_t = img2tensor(
-                cropped_face / 255.0, bgr2rgb=True, float32=True
-            )
-            normalize(cropped_face_t, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5), inplace=True)
-            cropped_face_t = cropped_face_t.unsqueeze(0).to(self.device)
-
-            try:
-                with torch.no_grad():
-                    output = self.net(
-                        cropped_face_t, w=codeformer_fidelity, adain=True
-                    )[0]
-                    restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1))
-                del output
-                torch.cuda.empty_cache()
-            except Exception as error:
-                print(f"\tFailed inference for CodeFormer: {error}")
-                restored_face = tensor2img(
-                    cropped_face_t, rgb2bgr=True, min_max=(-1, 1)
-                )
-
-            restored_face = restored_face.astype("uint8")
-            self.face_helper.add_restored_face(restored_face)
-
-        # paste_back
-        if not has_aligned:
-            # upsample the background
-            if bg_upsampler is not None:
-                # Now only support RealESRGAN for upsampling background
-                bg_img = bg_upsampler.enhance(img, outscale=upscale)[0]
-            else:
-                bg_img = None
-            self.face_helper.get_inverse_affine(None)
-            # paste each restored face to the input image
-            if face_upsample and face_upsampler is not None:
-                restored_img = self.face_helper.paste_faces_to_input_image(
-                    upsample_img=bg_img,
-                    draw_box=draw_box,
-                    face_upsampler=face_upsampler,
-                )
-            else:
-                restored_img = self.face_helper.paste_faces_to_input_image(
-                    upsample_img=bg_img, draw_box=draw_box
-                )
-
-        # save restored img
-        out_path = Path(tempfile.mkdtemp()) / 'output.png'
-        imwrite(restored_img, str(out_path))
-
-        return out_path
-
-
-def imread(img_path):
-    img = cv2.imread(img_path)
-    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
-    return img
-
-
-def set_realesrgan():
-    if not torch.cuda.is_available():  # CPU
-        import warnings
-
-        warnings.warn(
-            "The unoptimized RealESRGAN is slow on CPU. We do not use it. "
-            "If you really want to use it, please modify the corresponding codes.",
-            category=RuntimeWarning,
-        )
-        upsampler = None
-    else:
-        model = RRDBNet(
-            num_in_ch=3,
-            num_out_ch=3,
-            num_feat=64,
-            num_block=23,
-            num_grow_ch=32,
-            scale=2,
-        )
-        upsampler = RealESRGANer(
-            scale=2,
-            model_path="./weights/realesrgan/RealESRGAN_x2plus.pth",
-            model=model,
-            tile=400,
-            tile_pad=40,
-            pre_pad=0,
-            half=True,
-        )
-    return upsampler
@@ -1,3 +0,0 @@
-# Weights
-
-Put the downloaded pre-trained models to this folder.
@@ -50,7 +50,7 @@ requests==2.31.0
 tqdm==4.66.1
 accelerate==0.20.3
 opencv-python-headless==4.7.0.72
-diffusers==0.22.0
+diffusers==0.23.0
 einops==0.4.1
 gradio==3.43.2
 huggingface_hub==0.18.0
@@ -16,7 +16,7 @@ class ScriptPostprocessingUpscale(scripts_postprocessing.ScriptPostprocessing):
            with FormRow(elem_id="extras_upscale"):
                with gr.Tabs(elem_id="extras_resize_mode"):
                    with gr.TabItem('Scale by', elem_id="extras_scale_by_tab") as tab_scale_by:
-                        upscaling_resize = gr.Slider(minimum=1.0, maximum=8.0, step=0.05, label="Resize", value=4, elem_id="extras_upscaling_resize")
+                        upscaling_resize = gr.Slider(minimum=1.0, maximum=8.0, step=0.05, label="Resize", value=2.0, elem_id="extras_upscaling_resize")

                    with gr.TabItem('Scale to', elem_id="extras_scale_to_tab") as tab_scale_to:
                        with FormRow():
@@ -18,7 +18,7 @@ class Script(scripts.Script):
        info = gr.HTML("<p style=\"margin-bottom:0.75em\">Will upscale the image by the selected scale factor; use width and height sliders to set tile size</p>")
        overlap = gr.Slider(minimum=0, maximum=256, step=16, label='Tile overlap', value=64, elem_id=self.elem_id("overlap"))
        scale_factor = gr.Slider(minimum=1.0, maximum=4.0, step=0.05, label='Scale Factor', value=2.0, elem_id=self.elem_id("scale_factor"))
-        upscaler_index = gr.Radio(label='Upscaler', choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name, type="index", elem_id=self.elem_id("upscaler_index"))
+        upscaler_index = gr.Dropdown(label='Upscaler', choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name, type="index", elem_id=self.elem_id("upscaler_index"))

        return [info, overlap, upscaler_index, scale_factor]

@@ -86,6 +86,17 @@ def apply_checkpoint(p, x, xs):
        p.override_settings['sd_model_checkpoint'] = info.name


+def apply_refiner(p, x, xs):
+    if x == shared.opts.sd_model_refiner:
+        return
+    info = sd_models.get_closet_checkpoint_match(x)
+    if info is None:
+        shared.log.warning(f"XYZ grid: apply refiner unknown checkpoint: {x}")
+    else:
+        sd_models.reload_model_weights(shared.sd_refiner, info)
+        p.override_settings['sd_model_refiner'] = info.name
+
+
 def apply_dict(p, x, xs):
    if x == shared.opts.sd_model_dict:
        return
@@ -240,11 +251,12 @@ axis_options = [
    AxisOption("[Second pass] upscaler", str, apply_field("hr_upscaler"), choices=lambda: [*shared.latent_upscale_modes, *[x.name for x in shared.sd_upscalers]]),
    AxisOption("[Second pass] sampler", str, apply_latent_sampler, fmt=format_value, confirm=confirm_samplers, choices=lambda: [x.name for x in sd_samplers.samplers]),
    AxisOption("[Second pass] denoising Strength", float, apply_field("denoising_strength")),
-    AxisOption("[Second pass] steps", int, apply_field("hr_second_pass_steps")),
+    AxisOption("[Second pass] hires steps", int, apply_field("hr_second_pass_steps")),
    AxisOption("[Second pass] CFG scale", float, apply_field("image_cfg_scale")),
    AxisOption("[Second pass] guidance rescale", float, apply_field("diffusers_guidance_rescale")),
-    AxisOption("[Second pass] refiner start", float, apply_field("refiner_start")),
-    AxisOption("[Second pass] refiner start", float, apply_field("refiner_start")),
+    AxisOption("[Refiner] model", str, apply_refiner, fmt=format_value, cost=1.0, choices=lambda: sorted(sd_models.checkpoints_list)),
+    AxisOption("[Refiner] refiner start", float, apply_field("refiner_start")),
+    AxisOption("[Refiner] refiner steps", float, apply_field("refiner_steps")),
    AxisOption("[TOME] Token merging ratio (txt2img)", float, apply_override('token_merging_ratio')),
    AxisOption("[TOME] Token merging ratio (hires)", float, apply_override('token_merging_ratio_hr')),
    AxisOption("[FreeU] 1st stage backbone factor", float, apply_setting('freeu_b1')),
@@ -156,7 +156,7 @@ def initialize():


 def load_model():
-    if opts.sd_checkpoint_autoload:
+    if opts.sd_checkpoint_autoload and (shared.cmd_opts.ckpt is not None and shared.cmd_opts.ckpt.lower() != 'none'):
        shared.state.begin('load')
        thread_model = Thread(target=lambda: shared.sd_model)
        thread_model.start()