Merge branch 'dev' into master

.pylintrc

@@ -14,9 +14,8 @@ ignore-paths=/usr/lib/.*$,
              ^extensions/.*$,
              ^extensions-builtin/.*$,
              ^modules/dml/.*$,
-             ^modules/models/diffusion/.*$,
-             ^modules/xadapters/.*$,
              ^modules/tcd/.*$,
+             ^modules/xadapters/.*$,
 ignore-patterns=
 ignored-modules=
 jobs=0

.vscode/settings.json

@@ -2,12 +2,10 @@
   "python.analysis.extraPaths": [
     ".",
     "./modules",
-    "./repositories/BLIP",
-    "./repositories/CodeFormer",
-    "./repositories/k-diffusion",
-    "./repositories/taming-transformers",
-    "./repositories/stable-diffusion-stability-ai",
-    "./repositories/stable-diffusion-stability-ai/ldm"
+    "./repositories/blip",
+    "./repositories/codeformer",
+    "./repositories/ldm",
+    "./repositories/taming"
   ],
   "python.analysis.typeCheckingMode": "off",
   "editor.formatOnSave": false

CHANGELOG.md

@@ -2,30 +2,51 @@
 
 ## TODO
 
-- EDM samplers for Playground require `diffusers==0.27.0`
-- StableCascade requires diffusers `kashif/diffusers.git@wuerstchen-v3`
+- items that require `diffusers==0.27.0.dev`:
+  - EDM samplers for Playground 2.5
+  - Stable Cascade
 
-## Update for 2024-03-01
+## Update for 2024-03-08
 
 - [Playground v2.5](https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic)
   - new model version from Playground: based on SDXL, but with some cool new concepts
   - download using networks -> reference
   - set sampler to *DPM++ 2M EDM* or *Euler EDM*
 - [KOALA 700M](https://github.com/youngwanLEE/sdxl-koala)
-  - another very fast & light sd-xl model where original unet was compressed and distilled to 54% of original size  
+  - another very fast & light sdxl model where original unet was compressed and distilled to 54% of original size  
   - download using networks -> reference
   - *note* to download fp16 variant (recommended), set settings -> diffusers -> preferred model variant  
-- **Image2Video**
-  - new module for creating videos from images  
-  - simply enable from *img2img -> scripts -> image2video*  
-  - based on [VGen](https://huggingface.co/ali-vilab/i2vgen-xl)  
-- **VQA** visual question & answer in interrogate  
-  - with support for multiple variations of base models: *GIT, BLIP, ViLT, PIX*
+  [Stable Cascade](https://github.com/Stability-AI/StableCascade)
+  - large multi-stage high-quality model
+  - download using networks -> reference
+  - see [wiki](https://github.com/vladmandic/automatic/wiki/Stable-Cascade) for details
+  - currently requires 10GB VRAM, lighter version is in development
+- **Visual Query** visual query & answer in process tab  
+  - go to process -> visual query  
+  - ask your questions, e.g. "describe the image", "what is behind the subject", "what are predominant colors of the image?"
+  - primary model is [moondream2](https://github.com/vikhyat/moondream), a *tiny* 1.86B vision language model  
+    *note*: its still 3.7GB in size, so not really tiny  
+  - additional support for multiple variations of several base models: *GIT, BLIP, ViLT, PIX*, sizes range from 0.3 to 1.7GB  
 - **Second Pass / Refine**
   - independent upscale and hires options: run hires without upscale or upscale without hires or both
   - upscale can now run 0.1-8.0 scale and will also run if enabled at 1.0 to allow for upscalers that simply improve image quality
   - update ui section to reflect changes
   - *note*: behavior using backend:original is unchanged for backwards compatibilty
+- **Image2Video**
+  - new module for creating videos from images  
+  - simply enable from *img2img -> scripts -> image2video*  
+  - based on [VGen](https://huggingface.co/ali-vilab/i2vgen-xl)  
+- **Composable LoRA**, thanks @AI-Casanova
+  - control lora strength for each step
+    for example: `<xxx:0.1@0,0.9@1>` means strength=0.1 for step at 0% and intepolate towards strength=0.9 for step at 100%
+  - set sampler to *Composable LoRA*  
+  - *note*: this is a very experimental feature and may not work as expected
+- **Control**
+  - added *refiner/hires* workflows
+- **ROCm**  
+  - added *flash attention* support for rdna3, thanks @Disty0  
+    install flash_attn package for rdna3 manually and enable *flash attention* from *compute settings*   
+    to install flash_attn, activate the venv and run `pip install -U git+https://github.com/ROCm/flash-attention@howiejay/navi_support`   
 - **Samplers**
   - [TCD](https://mhh0318.github.io/tcd/): Trajectory Consistency Distillation  
     new sampler that produces consistent results in a very low number of steps (comparable to LCM but without reliance on LoRA)  
@@ -33,26 +54,62 @@
   - *DPM++ 2M EDM* and *Euler EDM*  
     EDM is a new solver algorithm currently available for DPM++2M and Euler samplers  
     Note that using EDM samplers with non-EDM optimized models will provide just noise and vice-versa  
+- **Styles**: styles are not just for prompts!
+  - new styles editor: *networks -> styles -> edit*
+  - styles can apply generate parameters, for example to have a style that enables and configures hires:  
+    parameters=`enable_hr: True, hr_scale: 2, hr_upscaler: Latent Bilinear antialias, hr_sampler_name: DEIS, hr_second_pass_steps: 20, denoising_strength: 0.5`
+  - styles can apply wildcards to prompts, for example:  
+    wildcards=`movie=mad max, dune, star wars, star trek; intricate=realistic, color sketch, pencil sketch, intricate`
+  - as usual, you can apply any number of styles so you can choose which settings are applied and in which order and which wildcards are used
+- **UI**
+  - *aspect-ratio** add selector and lock to width/height control  
+    allowed aspect ration can be configured via *settings -> user interface*  
+  - *interrogate* tab is now merged into *process* tab  
+  - *image viewer* now displays image metadata
+  - *themes* improve on-the-fly switching
+  - *log monitor* flag server warnings/errors and overall improve display
+  - *control* separate processor settings from unit settings
+- **Watermarking**
+  - SD.Next disables all known watermarks in models, but does allow user to set custom watermark  
+  - See *settings -> image options -> watermarking*  
+  - Invisible watermark: using steganogephy  
+  - Image watermark: overlaid on top of image  
 - **Improvements**
   - **FaceID** extend support for LoRA, HyperTile and FreeU, thanks @Trojaner
   - **Tiling** now extends to both Unet and VAE producing smoother outputs, thanks @AI-Casanova
   - new setting in image options: *include mask in output*
+  - improved params parsing from from prompt string and styles
   - default theme updates and additional built-in theme *black-gray*
   - add **ROCm** 6.0 nightly option to installer, thanks @jicka
   - support models with their own YAML model config files
   - support models with their own JSON per-component config files, for example: `playground-v2.5_vae.config`
+- **API**
+  - add preprocessor api endpoints  
+    GET:`/sdapi/v1/preprocessors`, POST:`/sdapi/v1/preprocess`, sample script:`cli/simple-preprocess.py`
+  - add masking api endpoints  
+    GET:`/sdapi/v1/masking`, POST:`/sdapi/v1/mask`, sample script:`cli/simple-mask.py`
 - **Internal**
+  - **stable-fast** compatibility with torch 2.2.1  
   - remove obsolete textual inversion training code
   - remove obsolete hypernetworks training code
+- **Refiner** validated workflows:
+  - Fully functional: SD15 + SD15, SDXL + SDXL, SDXL + SDXL-R
+  - Functional, but result is not as good: SD15 + SDXL, SDXL + SD15, SD15 + SDXL-R
+- **SDXL Lightning** models just-work, just makes sure to set CFG Scale to 0  
+    and choose a best-suited sampler, it may not be the one you're used to (e.g. maybe even basic Euler)  
 - **Fixes**
-  - improve model cpu offload compatibility
-  - improve model sequential offload compatibility
-  - improve bfloat16 compatibility
+  - improve *model cpu offload* compatibility
+  - improve *model sequential offload* compatibility
+  - improve *bfloat16* compatibility
+  - improve *xformers* installer to match cuda version and install triton
   - fix extra networks refresh
-  - fix sdp memory attention in backend original
+  - fix *sdp memory attention* in backend original
   - fix autodetect sd21 models
   - fix api info endpoint
-  - fix sampler eta in xyz grid, thanks @AI-Casanova
+  - fix *sampler eta* in xyz grid, thanks @AI-Casanova
+  - fix *requires_aesthetics_score* errors
+  - fix t2i-canny
+  - use diffusers lora load override for *lcm/tcd/turbo loras*
   - exception handler around vram memory stats gather
   - improve ZLUDA installer with `--use-zluda` cli param, thanks @lshqqytiger
 
@@ -293,7 +350,7 @@ Further details:
     - full implementation for *SD15* and *SD-XL*, to use simply select from *Scripts*  
       **Base** (93MB) uses *InsightFace* to generate face embeds and *OpenCLIP-ViT-H-14* (2.5GB) as image encoder  
       **Plus** (150MB) uses *InsightFace* to generate face embeds and *CLIP-ViT-H-14-laion2B* (3.8GB) as image encoder  
-      **SXDL** (1022MB) uses *InsightFace* to generate face embeds and *OpenCLIP-ViT-bigG-14* (3.7GB) as image encoder  
+      **SDXL** (1022MB) uses *InsightFace* to generate face embeds and *OpenCLIP-ViT-bigG-14* (3.7GB) as image encoder  
   - [FaceSwap](https://github.com/deepinsight/insightface/blob/master/examples/in_swapper/README.md)  
     - face swap performs face swapping at the end of generation  
     - based on InsightFace in-swapper  
@@ -310,7 +367,7 @@ Further details:
 - [IPAdapter](https://huggingface.co/h94/IP-Adapter)  
   - additional models for *SD15* and *SD-XL*, to use simply select from *Scripts*:  
     **SD15**: Base, Base ViT-G, Light, Plus, Plus Face, Full Face  
-    **SDXL**: Base SXDL, Base ViT-H SXDL, Plus ViT-H SXDL, Plus Face ViT-H SXDL  
+    **SDXL**: Base SDXL, Base ViT-H SDXL, Plus ViT-H SDXL, Plus Face ViT-H SDXL  
   - enable use via api, thanks @trojaner  
 - [Segmind SegMoE](https://github.com/segmind/segmoe)  
   - initial support for reference models  

TODO.md

@@ -9,7 +9,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
 - ipadapter masking: <https://github.com/huggingface/diffusers/pull/6847>
 - x-adapter: <https://github.com/showlab/X-Adapter>
 - async lowvram: <https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/14855>
-- init latents: variations, tiling, img2img
+- init latents: variations, img2img
 - diffusers public callbacks  
 - remove builtin: controlnet
 - remove builtin: image-browser
@@ -18,5 +18,3 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
 
 - second pass: <https://github.com/vladmandic/automatic/issues/2783>  
 - control api  
-- masking api  
-- preprocess api

cli/install-sf.py

@@ -3,7 +3,7 @@ import os
 import re
 import sys
 
-torch_supported = ['211', '212']
+torch_supported = ['211', '212','220','221']
 cuda_supported = ['cu118', 'cu121']
 python_supported = ['39', '310', '311']
 repo_url = 'https://github.com/chengzeyi/stable-fast'

cli/simple-mask.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+import io
+import os
+import time
+import base64
+import logging
+import argparse
+import requests
+import urllib3
+from PIL import Image
+
+
+sd_url = os.environ.get('SDAPI_URL', "http://127.0.0.1:7860")
+sd_username = os.environ.get('SDAPI_USR', None)
+sd_password = os.environ.get('SDAPI_PWD', None)
+
+
+logging.basicConfig(level = logging.INFO, format = '%(asctime)s %(levelname)s: %(message)s')
+log = logging.getLogger(__name__)
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+def auth():
+    if sd_username is not None and sd_password is not None:
+        return requests.auth.HTTPBasicAuth(sd_username, sd_password)
+    return None
+
+
+def get(endpoint: str, dct: dict = None):
+    req = requests.get(f'{sd_url}{endpoint}', json=dct, timeout=300, verify=False, auth=auth())
+    if req.status_code != 200:
+        return { 'error': req.status_code, 'reason': req.reason, 'url': req.url }
+    else:
+        return req.json()
+
+
+def post(endpoint: str, dct: dict = None):
+    req = requests.post(f'{sd_url}{endpoint}', json = dct, timeout=300, verify=False, auth=auth())
+    if req.status_code != 200:
+        return { 'error': req.status_code, 'reason': req.reason, 'url': req.url }
+    else:
+        return req.json()
+
+
+def info(args): # pylint: disable=redefined-outer-name
+    t0 = time.time()
+    with open(args.input, 'rb') as f:
+        image = base64.b64encode(f.read()).decode()
+    if args.mask:
+        with open(args.mask, 'rb') as f:
+            mask = base64.b64encode(f.read()).decode()
+    else:
+        mask = None
+    options = get('/sdapi/v1/masking')
+    log.info(f'options: {options}')
+    req = {
+        'image': image,
+        'mask': mask,
+        'type': args.type or 'Composite',
+        'params': { 'auto_mask': 'Grayscale' if mask is None else None },
+    }
+    data = post('/sdapi/v1/mask', req)
+    t1 = time.time()
+    if 'mask' in data:
+        b64 = data['mask'].split(',',1)[0]
+        image = Image.open(io.BytesIO(base64.b64decode(b64)))
+        log.info(f'received image: size={image.size} time={t1-t0:.2f}')
+        if args.output:
+            image.save(args.output)
+            log.info(f'saved image: fn={args.output}')
+    else:
+        log.info(f'received: {data} time={t1-t0:.2f}')
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description = 'simple-info')
+    parser.add_argument('--input', required=True, help='input image')
+    parser.add_argument('--mask', required=False, help='input mask')
+    parser.add_argument('--type', required=False, help='output mask type')
+    parser.add_argument('--output', required=False, help='output image')
+    args = parser.parse_args()
+    log.info(f'info: {args}')
+    info(args)

cli/simple-preprocess.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+import io
+import os
+import time
+import base64
+import logging
+import argparse
+import requests
+import urllib3
+from PIL import Image
+
+
+sd_url = os.environ.get('SDAPI_URL', "http://127.0.0.1:7860")
+sd_username = os.environ.get('SDAPI_USR', None)
+sd_password = os.environ.get('SDAPI_PWD', None)
+
+
+logging.basicConfig(level = logging.INFO, format = '%(asctime)s %(levelname)s: %(message)s')
+log = logging.getLogger(__name__)
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+def auth():
+    if sd_username is not None and sd_password is not None:
+        return requests.auth.HTTPBasicAuth(sd_username, sd_password)
+    return None
+
+
+def get(endpoint: str, dct: dict = None):
+    req = requests.get(f'{sd_url}{endpoint}', json=dct, timeout=300, verify=False, auth=auth())
+    if req.status_code != 200:
+        return { 'error': req.status_code, 'reason': req.reason, 'url': req.url }
+    else:
+        return req.json()
+
+
+def post(endpoint: str, dct: dict = None):
+    req = requests.post(f'{sd_url}{endpoint}', json = dct, timeout=300, verify=False, auth=auth())
+    if req.status_code != 200:
+        return { 'error': req.status_code, 'reason': req.reason, 'url': req.url }
+    else:
+        return req.json()
+
+
+def info(args): # pylint: disable=redefined-outer-name
+    t0 = time.time()
+    with open(args.input, 'rb') as f:
+        content = f.read()
+    models = get('/sdapi/v1/preprocessors')
+    log.info(f'models: {models}')
+    req = {
+        'model': args.model or 'Canny',
+        'image': base64.b64encode(content).decode(),
+        'config': { 'low_threshold': 50 },
+    }
+    data = post('/sdapi/v1/preprocess', req)
+    t1 = time.time()
+    if 'image' in data:
+        b64 = data['image'].split(',',1)[0]
+        image = Image.open(io.BytesIO(base64.b64decode(b64)))
+        log.info(f'received image: size={image.size} time={t1-t0:.2f}')
+        if args.output:
+            image.save(args.output)
+            log.info(f'saved image: fn={args.output}')
+    else:
+        log.info(f'received: {data} time={t1-t0:.2f}')
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description = 'simple-info')
+    parser.add_argument('--input', required=True, help='input image')
+    parser.add_argument('--model', required=True, help='preprocessing model')
+    parser.add_argument('--output', required=False, help='output image')
+    args = parser.parse_args()
+    log.info(f'info: {args}')
+    info(args)

extensions-builtin/Lora/extra_networks_lora.py

@@ -1,8 +1,43 @@
 import time
+import numpy as np
+import re
 import networks
 import lora_patches
 from modules import extra_networks, shared
 
+# from https://github.com/cheald/sd-webui-loractl/blob/master/loractl/lib/utils.py
+def get_stepwise(param, step, steps):
+    def sorted_positions(raw_steps):
+        steps = [[float(s.strip()) for s in re.split("[@~]", x)]
+                 for x in re.split("[,;]", str(raw_steps))]
+        # If we just got a single number, just return it
+        if len(steps[0]) == 1:
+            return steps[0][0]
+
+        # Add implicit 1s to any steps which don't have a weight
+        steps = [[s[0], s[1] if len(s) == 2 else 1] for s in steps]
+
+        # Sort by index
+        steps.sort(key=lambda k: k[1])
+
+        steps = [list(v) for v in zip(*steps)]
+        return steps
+
+    def calculate_weight(m, step, max_steps, step_offset=2):
+        if isinstance(m, list):
+            if m[1][-1] <= 1.0:
+                if max_steps > 0:
+                    step = (step) / (max_steps - step_offset)
+                else:
+                    step = 1.0
+            else:
+                step = step
+            v = np.interp(step, m[1], m[0])
+            return v
+        else:
+            return m
+    return calculate_weight(sorted_positions(param), step, steps)
+
 
 class ExtraNetworkLora(extra_networks.ExtraNetwork):
 
@@ -14,7 +49,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
 
         """mapping of network names to the number of errors the network had during operation"""
 
-    def activate(self, p, params_list):
+    def activate(self, p, params_list, step=0):
         t0 = time.time()
         self.errors.clear()
         if len(params_list) > 0:
@@ -29,13 +64,21 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
         for params in params_list:
             assert params.items
             names.append(params.positional[0])
-            te_multiplier = float(params.positional[1]) if len(params.positional) > 1 else 1.0
-            te_multiplier = float(params.named.get("te", te_multiplier))
-            unet_multiplier = [float(params.positional[2]) if len(params.positional) > 2 else te_multiplier] * 3
-            unet_multiplier = [float(params.named.get("unet", unet_multiplier[0]))] * 3
-            unet_multiplier[0] = float(params.named.get("in", unet_multiplier[0]))
-            unet_multiplier[1] = float(params.named.get("mid", unet_multiplier[1]))
-            unet_multiplier[2] = float(params.named.get("out", unet_multiplier[2]))
+            te_multiplier = params.named.get("te", params.positional[1] if len(params.positional) > 1 else 1.0)
+            if isinstance(te_multiplier, str) and "@" in te_multiplier:
+                te_multiplier = get_stepwise(te_multiplier, step, p.steps)
+            else:
+                te_multiplier = float(te_multiplier)
+            unet_multiplier = [params.positional[2] if len(params.positional) > 2 else te_multiplier] * 3
+            unet_multiplier = [params.named.get("unet", unet_multiplier[0])] * 3
+            unet_multiplier[0] = params.named.get("in", unet_multiplier[0])
+            unet_multiplier[1] = params.named.get("mid", unet_multiplier[1])
+            unet_multiplier[2] = params.named.get("out", unet_multiplier[2])
+            for i in range(len(unet_multiplier)):
+                if isinstance(unet_multiplier[i], str) and "@" in unet_multiplier[i]:
+                    unet_multiplier[i] = get_stepwise(unet_multiplier[i], step, p.steps)
+                else:
+                    unet_multiplier[i] = float(unet_multiplier[i])
             dyn_dim = int(params.positional[3]) if len(params.positional) > 3 else None
             dyn_dim = int(params.named["dyn"]) if "dyn" in params.named else dyn_dim
             te_multipliers.append(te_multiplier)
@@ -57,7 +100,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
                 network_hashes.append(f"{alias}: {shorthash}")
             if network_hashes:
                 p.extra_generation_params["Lora hashes"] = ", ".join(network_hashes)
-        if len(names) > 0:
+        if len(names) > 0 and step == 0:
             shared.log.info(f'LoRA apply: {names} patch={t1-t0:.2f} load={t2-t1:.2f}')
         elif self.active:
             self.active = False

extensions-builtin/Lora/networks.py

@@ -193,10 +193,13 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No
             try:
                 if recompile_model:
                     shared.compiled_model_state.lora_model.append(f"{name}:{te_multipliers[i] if te_multipliers else 1.0}")
-                if shared.backend == shared.Backend.DIFFUSERS and shared.opts.lora_force_diffusers: # OpenVINO only works with Diffusers LoRa loading.
-                    # or getattr(network_on_disk, 'shorthash', '').lower() == 'aaebf6360f7d' # sd15-lcm
-                    # or getattr(network_on_disk, 'shorthash', '').lower() == '3d18b05e4f56' # sdxl-lcm
-                    # or getattr(network_on_disk, 'shorthash', '').lower() == '813ea5fb1c67' # turbo sdxl-turbo
+                shorthash = getattr(network_on_disk, 'shorthash', '').lower()
+                if shared.backend == shared.Backend.DIFFUSERS and (shared.opts.lora_force_diffusers # OpenVINO only works with Diffusers LoRa loading.
+                        or shorthash == 'aaebf6360f7d' # sd15-lcm
+                        or shorthash == '3d18b05e4f56' # sdxl-lcm
+                        or shorthash == 'b71dcb732467' # sdxl-tcd
+                        or shorthash == '813ea5fb1c67' # sdxl-turbo
+                    ):
                     net = load_diffusers(name, network_on_disk, lora_scale=te_multipliers[i] if te_multipliers else 1.0)
                 else:
                     net = load_network(name, network_on_disk)

html/locale_en.json

@@ -165,7 +165,7 @@
   {"id":"","label":"Separate Init Image","localized":"","hint":"Creates an additional window next to Control input labeled Init input, so you can have a separate image for both Control operations and an init source."}
 ],
 "process tab": [
-  {"id":"","label":"Single Image","localized":"","hint":"Process single image"},
+  {"id":"","label":"Process Image","localized":"","hint":"Process single image"},
   {"id":"","label":"Process Batch","localized":"","hint":"Process batch of images"},
   {"id":"","label":"Process Folder","localized":"","hint":"Process all images in a folder"},
   {"id":"","label":"Scale by","localized":"","hint":"Use this tab to resize the source image(s) by a chosen factor"},

html/reference.json

@@ -3,22 +3,26 @@
     "path": "dreamshaper_8.safetensors@https://civitai.com/api/download/models/128713",
     "desc": "Showcase finetuned model based on Stable diffusion 1.5",
     "preview": "dreamshaper_8.jpg",
+    "resolution": 512,
     "original": true
   },
   "DreamShaper SD XL Turbo": {
     "path": "dreamshaperXL_turboDpmppSDE.safetensors@https://civitai.com/api/download/models/251662",
     "desc": "Showcase finetuned model based on Stable diffusion XL",
+    "resolution": 1024,
     "preview": "dreamshaperXL_turboDpmppSDE.jpg"
   }, 
   "Juggernaut Reborn": {
     "path": "juggernaut_reborn.safetensors@https://civitai.com/api/download/models/274039",
     "desc": "Showcase finetuned model based on Stable diffusion 1.5",
     "preview": "juggernaut_reborn.jpg",
+    "resolution": 512,
     "original": true
   },
   "Juggernaut XL v7 RunDiffusion": {
     "path": "juggernautXL_v7Rundiffusion.safetensors@https://civitai.com/api/download/models/240840",
     "desc": "Showcase finetuned model based on Stable diffusion XL",
+    "resolution": 1024,
     "preview": "juggernautXL_v7Rundiffusion.jpg"
   },
   "RunwayML SD 1.5": {
@@ -26,6 +30,7 @@
     "alt": "v1-5-pruned-emaonly.safetensors@https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensors?download=true",
     "desc": "Stable Diffusion 1.5 is the base model all other 1.5 checkpoint were trained from. It's a latent text-to-image diffusion model capable of generating photo-realistic images given any text input. The Stable-Diffusion-v1-5 checkpoint was initialized with the weights of the Stable-Diffusion-v1-2 checkpoint and subsequently fine-tuned on 595k steps at resolution 512x512.",
     "preview": "runwayml--stable-diffusion-v1-5.jpg",
+    "resolution": 512,
     "original": true
   },
   "StabilityAI SD 2.1 EMA": {
@@ -33,6 +38,7 @@
     "alt": "v2-1_512-ema-pruned.safetensors@https://huggingface.co/stabilityai/stable-diffusion-2-1-base/resolve/main/v2-1_512-ema-pruned.safetensors?download=true",
     "desc": "This stable-diffusion-2-1-base model fine-tunes stable-diffusion-2-base (512-base-ema.ckpt) with 220k extra steps taken",
     "preview": "stabilityai--stable-diffusion-2.1-base.jpg",
+    "resolution": 512,
     "original": true
   },
   "StabilityAI SD 2.1 V": {
@@ -40,12 +46,15 @@
     "alt": "v2-1_768-ema-pruned.safetensors@https://huggingface.co/stabilityai/stable-diffusion-2-1/resolve/main/v2-1_768-ema-pruned.safetensors?download=true",
     "desc": "This stable-diffusion-2 model is resumed from stable-diffusion-2-base (512-base-ema.ckpt) and trained for 150k steps using a v-objective on the same dataset. Resumed for another 140k steps on 768x768 images",
     "preview": "stabilityai--stable-diffusion-2.1-base.jpg",
+    "resolution": 768,
     "original": true
   },
   "StabilityAI SD-XL 1.0 Base": {
-    "path": "stabilityai/stable-diffusion-xl-base-1.0",
+    "path": "huggingface/stabilityai/stable-diffusion-xl-base-1.0",
+    "skip": true,
     "variant": "fp16",
     "desc": "Stable Diffusion XL (SDXL) is the latest AI image generation model that is tailored towards more photorealistic outputs with more detailed imagery and composition compared to previous SD models, including SD 2.1. It can make realistic faces, legible text within the images, and better image composition, all while using shorter and simpler prompts at a greatly increased base resolution of 1024x1024. Just like its predecessors, SDXL has the ability to generate image variations using image-to-image prompting, inpainting (reimagining of the selected parts of an image), and outpainting (creating new parts that lie outside the image borders).",
+    "resolution": 1024,
     "preview": "stabilityai--stable-diffusion-xl-base-1.0.jpg"
   },
   "StabilityAI SD 2.1 Turbo": {
@@ -54,6 +63,7 @@
     "variant": "fp16",
     "desc": "SD-Turbo is a distilled version of Stable Diffusion 2.1, trained for real-time synthesis. SD-Turbo is based on a novel training method called Adversarial Diffusion Distillation (ADD) (see the technical report), which allows sampling large-scale foundational image diffusion models in 1 to 4 steps at high image quality. This approach uses score distillation to leverage large-scale off-the-shelf image diffusion models as a teacher signal and combines this with an adversarial loss to ensure high image fidelity even in the low-step regime of one or two sampling steps.",
     "preview": "stabilityai--sd-turbo.jpg",
+    "resolution": 512,
     "original": true
   },
   "StabilityAI SD-XL Turbo": {
@@ -61,6 +71,7 @@
     "path": "sdxl_turbo.safetensors@https://huggingface.co/stabilityai/sdxl-turbo/resolve/main/sd_xl_turbo_1.0_fp16.safetensors?download=true",
     "variant": "fp16",
     "desc": "SDXL-Turbo is a distilled version of SDXL 1.0, trained for real-time synthesis. SDXL-Turbo is based on a novel training method called Adversarial Diffusion Distillation (ADD) (see the technical report), which allows sampling large-scale foundational image diffusion models in 1 to 4 steps at high image quality. This approach uses score distillation to leverage large-scale off-the-shelf image diffusion models as a teacher signal and combines this with an adversarial loss to ensure high image fidelity even in the low-step regime of one or two sampling steps.",
+    "resolution": 1024,
     "preview": "stabilityai--sdxl-turbo.jpg"
   },
   "StabilityAI Stable Video Diffusion": {
@@ -77,116 +88,139 @@
     "path": "huggingface/stabilityai/stable-cascade",
     "skip": true,
     "desc": "Stable Cascade is a diffusion model built upon the Würstchen architecture and its main difference to other models like Stable Diffusion is that it is working at a much smaller latent space. Why is this important? The smaller the latent space, the faster you can run inference and the cheaper the training becomes. How small is the latent space? Stable Diffusion uses a compression factor of 8, resulting in a 1024x1024 image being encoded to 128x128. Stable Cascade achieves a compression factor of 42, meaning that it is possible to encode a 1024x1024 image to 24x24, while maintaining crisp reconstructions. The text-conditional model is then trained in the highly compressed latent space. Previous versions of this architecture, achieved a 16x cost reduction over Stable Diffusion 1.5",
+    "resolution": 1024,
     "preview": "stabilityai--stable-cascade.jpg"
   },
   "Segmind Vega": {
     "path": "segmind/Segmind-Vega",
     "desc": "The Segmind-Vega Model is a distilled version of the Stable Diffusion XL (SDXL), offering a remarkable 70% reduction in size and an impressive 100% speedup while retaining high-quality text-to-image generation capabilities. Trained on diverse datasets, including Grit and Midjourney scrape data, it excels at creating a wide range of visual content based on textual prompts. Employing a knowledge distillation strategy, Segmind-Vega leverages the teachings of several expert models, including SDXL, ZavyChromaXL, and JuggernautXL, to combine their strengths and produce compelling visual outputs.",
+    "resolution": 1024,
     "preview": "segmind--Segmind-Vega.jpg"
   },
   "Segmind SSD-1B": {
     "path": "segmind/SSD-1B",
     "desc": "The Segmind Stable Diffusion Model (SSD-1B) offers a compact, efficient, and distilled version of the SDXL model. At 50% smaller and 60% faster than Stable Diffusion XL (SDXL), it provides quick and seamless performance without sacrificing image quality.",
+    "resolution": 1024,
     "preview": "segmind--SSD-1B.jpg"
   },
   "Segmind Tiny": {
     "path": "segmind/tiny-sd",
     "desc": "Segmind's Tiny-SD offers a compact, efficient, and distilled version of Realistic Vision 4.0 and is up to 80% faster than SD1.5",
+    "resolution": 512,
     "preview": "segmind--tiny-sd.jpg"
   },
   "Segmind SegMoE SD 4x2": {
     "path": "segmind/SegMoE-SD-4x2-v0",
     "desc": "SegMoE-SD-4x2-v0 is an untrained Segmind Mixture of Diffusion Experts Model generated using segmoe from 4 Expert SD1.5 models. SegMoE is a powerful framework for dynamically combining Stable Diffusion Models into a Mixture of Experts within minutes without training",
+    "resolution": 512,
     "preview": "segmind--SegMoE-SD-4x2-v0.jpg"
   },
   "Segmind SegMoE XL 2x1": {
     "path": "segmind/SegMoE-2x1-v0",
     "desc": "SegMoE-2x1-v0 is an untrained Segmind Mixture of Diffusion Experts Model generated using segmoe from 2 Expert SDXL models. SegMoE is a powerful framework for dynamically combining Stable Diffusion Models into a Mixture of Experts within minutes without training",
+    "resolution": 1024,
     "preview": "segmind--SegMoE-2x1-v0.jpg"
   },
   "Segmind SegMoE XL 4x2": {
     "path": "segmind/SegMoE-4x2-v0",
     "desc": "SegMoE-4x2-v0 is an untrained Segmind Mixture of Diffusion Experts Model generated using segmoe from 4 Expert SDXL models. SegMoE is a powerful framework for dynamically combining Stable Diffusion Models into a Mixture of Experts within minutes without training",
+    "resolution": 1024,
     "preview": "segmind--SegMoE-4x2-v0.jpg"
   },
   "LCM SD-1.5 Dreamshaper 7": {
     "path": "SimianLuo/LCM_Dreamshaper_v7",
     "desc": "Latent Consistencey Models enable swift inference with minimal steps on any pre-trained LDMs, including Stable Diffusion. By distilling classifier-free guidance into the model's input, LCM can generate high-quality images in very short inference time. LCM can generate quality images in as few as 3-4 steps, making it blazingly fast.",
+    "resolution": 512,
     "preview": "SimianLuo--LCM_Dreamshaper_v7.jpg"
   },
   "Pixart-α XL 2 Medium 512": {
     "path": "PixArt-alpha/PixArt-XL-2-512x512",
     "desc": "PixArt-α is a Transformer-based T2I diffusion model whose image generation quality is competitive with state-of-the-art image generators (e.g., Imagen, SDXL, and even Midjourney), and the training speed markedly surpasses existing large-scale T2I models. Extensive experiments demonstrate that PIXART-α excels in image quality, artistry, and semantic control. It can directly generate 512px images from text prompts within a single sampling process.",
+    "resolution": 512,
     "preview": "PixArt-alpha--PixArt-XL-2-512x512.jpg"
   },
   "Pixart-α XL 2 Large 1024": {
     "path": "PixArt-alpha/PixArt-XL-2-1024-MS",
     "desc": "PixArt-α is a Transformer-based T2I diffusion model whose image generation quality is competitive with state-of-the-art image generators (e.g., Imagen, SDXL, and even Midjourney), and the training speed markedly surpasses existing large-scale T2I models. Extensive experiments demonstrate that PIXART-α excels in image quality, artistry, and semantic control. It can directly generate 1024px images from text prompts within a single sampling process.",
+    "resolution": 1024,
     "preview": "PixArt-alpha--PixArt-XL-2-1024-MS.jpg"
   },
   "Pixart-α XL 2 Large LCM": {
     "path": "PixArt-alpha/PixArt-LCM-XL-2-1024-MS",
     "desc": "Pixart-α consists of pure transformer blocks for latent diffusion: It can directly generate 1024px images from text prompts within a single sampling process. LCMs is a diffusion distillation method which predict PF-ODE's solution directly in latent space, achieving super fast inference with few steps. Following LCM LoRA, we illustrative of the generation speed we achieve on various computers. Let us stress again how liberating it is to explore image generation so easily with PixArt-LCM.",
+    "resolution": 1024,
     "preview": "PixArt-alpha--PixArt-XL-2-1024-MS.jpg"
   },
   "Warp Wuerstchen": {
     "path": "warp-ai/wuerstchen",
     "desc": "Würstchen is a diffusion model whose text-conditional model works in a highly compressed latent space of images. Why is this important? Compressing data can reduce computational costs for both training and inference by magnitudes. Training on 1024x1024 images, is way more expensive than training at 32x32. Usually, other works make use of a relatively small compression, in the range of 4x - 8x spatial compression. Würstchen takes this to an extreme. Through its novel design, we achieve a 42x spatial compression. Würstchen employs a two-stage compression, what we call Stage A and Stage B. Stage A is a VQGAN, and Stage B is a Diffusion Autoencoder (more details can be found in the paper). A third model, Stage C, is learned in that highly compressed latent space. This training requires fractions of the compute used for current top-performing models, allowing also cheaper and faster inference.",
+    "resolution": 1024,
     "preview": "warp-ai--wuerstchen.jpg"
   },
   "Kandinsky 2.1": {
     "path": "kandinsky-community/kandinsky-2-1",
     "desc": "Kandinsky 2.1 is a text-conditional diffusion model based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder. Kandinsky 2.1 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas. It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.",
+    "resolution": 768,
     "preview": "kandinsky-community--kandinsky-2-1.jpg"
   },
   "Kandinsky 2.2": {
     "path": "kandinsky-community/kandinsky-2-2-decoder",
     "desc": "Kandinsky 2.2 is a text-conditional diffusion model (+0.1!) based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder. Kandinsky 2.1 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas. It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.",
+    "resolution": 768,
     "preview": "kandinsky-community--kandinsky-2-2-decoder.jpg"
   },
   "Kandinsky 3": {
     "path": "kandinsky-community/kandinsky-3",
     "desc": "Kandinsky 3.0 is an open-source text-to-image diffusion model built upon the Kandinsky2-x model family. In comparison to its predecessors, Kandinsky 3.0 incorporates more data and specifically related to Russian culture, which allows to generate pictures related to Russin culture. Furthermore, enhancements have been made to the text understanding and visual quality of the model, achieved by increasing the size of the text encoder and Diffusion U-Net models, respectively.",
+    "resolution": 1024,
     "preview": "kandinsky-community--kandinsky-3.jpg"
   },
   "Playground v1": {
     "path": "playgroundai/playground-v1",
     "desc": "Playground v1 is a latent diffusion model that improves the overall HDR quality to get more stunning images.",
+    "resolution": 512,
     "preview": "playgroundai--playground-v1.jpg"
   },
   "Playground v2 256": {
     "path": "playgroundai/playground-v2-256px-base",
     "desc": "Playground v2 is a diffusion-based text-to-image generative model. The model was trained from scratch by the research team at Playground. Images generated by Playground v2 are favored 2.5 times more than those produced by Stable Diffusion XL, according to Playground’s user study.",
+    "resolution": 256,
     "preview": "playgroundai--playground-v2-256px-base.jpg"
   },
   "Playground v2 512": {
     "path": "playgroundai/playground-v2-512px-base",
     "desc": "Playground v2 is a diffusion-based text-to-image generative model. The model was trained from scratch by the research team at Playground. Images generated by Playground v2 are favored 2.5 times more than those produced by Stable Diffusion XL, according to Playground’s user study.",
+    "resolution": 512,
     "preview": "playgroundai--playground-v2-512px-base.jpg"
   },
   "Playground v2 1024": {
     "path": "playgroundai/playground-v2-1024px-aesthetic",
     "desc": "Playground v2 is a diffusion-based text-to-image generative model. The model was trained from scratch by the research team at Playground. Images generated by Playground v2 are favored 2.5 times more than those produced by Stable Diffusion XL, according to Playground’s user study.",
+    "resolution": 1024,
     "preview": "playgroundai--playground-v2-1024px-aesthetic.jpg"
   },
   "Playground v2.5": {
     "path": "playground-v2.5-1024px-aesthetic.fp16.safetensors@https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic/resolve/main/playground-v2.5-1024px-aesthetic.fp16.safetensors?download=true",
     "desc": "Playground v2.5 is a diffusion-based text-to-image generative model, and a successor to Playground v2. Playground v2.5 is the state-of-the-art open-source model in aesthetic quality. Our user studies demonstrate that our model outperforms SDXL, Playground v2, PixArt-α, DALL-E 3, and Midjourney 5.2.",
+    "resolution": 1024,
     "preview": "playgroundai--playground-v2-1024px-aesthetic.jpg"
   },
   "DeepFloyd IF Medium": {
     "path": "DeepFloyd/IF-I-M-v1.0",
     "desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.",
+    "resolution": 1024,
     "preview": "DeepFloyd--IF-I-M-v1.0.jpg"
   },
   "aMUSEd 256": {
     "path": "amused/amused-256",
     "desc": "Amused is a lightweight text to image model based off of the muse architecture. Amused is particularly useful in applications that require a lightweight and fast model such as generating many images quickly at once.",
+    "resolution": 256,
     "preview": "amused--amused-256.jpg"
   },
   "aMUSEd 512": {
     "path": "amused/amused-512",
     "desc": "Amused is a lightweight text to image model based off of the muse architecture. Amused is particularly useful in applications that require a lightweight and fast model such as generating many images quickly at once.",
+    "resolution": 512,
     "preview": "amused--amused-512.jpg"
   },
   "KOALA 700M": {
@@ -194,12 +228,13 @@
     "variant": "fp16",
     "skip": true,
     "desc": "Fast text-to-image model, called KOALA, by compressing SDXL's U-Net and distilling knowledge from SDXL into our model. KOALA-700M can generate a 1024x1024 image in less than 1.5 seconds on an NVIDIA 4090 GPU, which is more than 2x faster than SDXL.",
+    "resolution": 1024,
     "preview": "etri-vilab--koala-700m-llava-cap.jpg"
   },
-  
   "Tsinghua UniDiffuser": {
     "path": "thu-ml/unidiffuser-v1",
     "desc": "UniDiffuser is a unified diffusion framework to fit all distributions relevant to a set of multi-modal data in one transformer. UniDiffuser is able to perform image, text, text-to-image, image-to-text, and image-text pair generation by setting proper timesteps without additional overhead.\nSpecifically, UniDiffuser employs a variation of transformer, called U-ViT, which parameterizes the joint noise prediction network. Other components perform as encoders and decoders of different modalities, including a pretrained image autoencoder from Stable Diffusion, a pretrained image ViT-B/32 CLIP encoder, a pretrained text ViT-L CLIP encoder, and a GPT-2 text decoder finetuned by ourselves.",
+    "resolution": 512,
     "preview": "thu-ml--unidiffuser-v1.jpg"
   },
   "SalesForce BLIP-Diffusion": {
@@ -210,6 +245,7 @@
   "InstaFlow 0.9B": {
     "path": "XCLiu/instaflow_0_9B_from_sd_1_5",
     "desc": "InstaFlow is an ultra-fast, one-step image generator that achieves image quality close to Stable Diffusion. This efficiency is made possible through a recent Rectified Flow technique, which trains probability flows with straight trajectories, hence inherently requiring only a single step for fast inference.",
+    "resolution": 512,
     "preview": "XCLiu--instaflow_0_9B_from_sd_1_5.jpg"
   }
 }

installer.py

@@ -67,6 +67,12 @@ def setup_logging():
             self.formatter = logging.Formatter('{ "asctime":"%(asctime)s", "created":%(created)f, "facility":"%(name)s", "pid":%(process)d, "tid":%(thread)d, "level":"%(levelname)s", "module":"%(module)s", "func":"%(funcName)s", "msg":"%(message)s" }')
 
         def emit(self, record):
+            if record.msg is not None and not isinstance(record.msg, str):
+                record.msg = str(record.msg)
+            try:
+                record.msg = record.msg.replace('"', "'")
+            except Exception:
+                pass
             msg = self.format(record)
             # self.buffer.append(json.loads(msg))
             self.buffer.append(msg)
@@ -425,9 +431,12 @@ def check_torch():
         log.info('nVidia CUDA toolkit detected: nvidia-smi present')
         if not args.use_xformers:
             torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/cu121')
+            xformers_package = os.environ.get('XFORMERS_PACKAGE', '--pre triton xformers --index-url https://download.pytorch.org/whl/cu121')
         else:
             torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/cu118')
-        xformers_package = os.environ.get('XFORMERS_PACKAGE', '--pre xformers' if opts.get('cross_attention_optimization', '') == 'xFormers' else 'none')
+            xformers_package = os.environ.get('XFORMERS_PACKAGE', '--pre triton xformers --index-url https://download.pytorch.org/whl/cu118')
+        if opts.get('cross_attention_optimization', '') != 'xFormers':
+            xformers_package = 'none'
         install('onnxruntime-gpu', 'onnxruntime-gpu', ignore=True)
     elif is_rocm_available():
         is_windows = platform.system() == 'Windows'
@@ -479,7 +488,6 @@ def check_torch():
             log.debug(f'ROCm hipconfig failed: {e}')
             rocm_ver = None
         if args.use_zluda:
-            torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.2.0 torchvision --index-url https://download.pytorch.org/whl/cu118')
             log.warning("ZLUDA support: experimental")
             zluda_need_dll_patch = is_windows and not installed('torch')
             zluda_path = find_zluda()
@@ -493,15 +501,24 @@ def check_torch():
                     import tarfile
                     archive_type = tarfile.TarFile
                     zluda_url = 'https://github.com/vosen/ZLUDA/releases/download/v3/zluda-3-linux.tar.gz'
-                urllib.request.urlretrieve(zluda_url, '_zluda')
-                with archive_type('_zluda', 'r') as f:
-                    f.extractall('.zluda')
-                zluda_path = os.path.abspath('./.zluda')
-                os.remove('_zluda')
-            log.debug(f'Found ZLUDA in {zluda_path}')
-            paths = os.environ.get('PATH', '.')
-            if zluda_path not in paths:
-                os.environ['PATH'] = paths + ';' + zluda_path
+                try:
+                    urllib.request.urlretrieve(zluda_url, '_zluda')
+                    with archive_type('_zluda', 'r') as f:
+                        f.extractall('.zluda')
+                    zluda_path = os.path.abspath('./.zluda')
+                    os.remove('_zluda')
+                except Exception as e:
+                    log.warning(f'Failed to install ZLUDA: {e}')
+            if os.path.exists(os.path.join(zluda_path, 'nvcuda.dll')):
+                log.info(f'Using ZLUDA in {zluda_path}')
+                torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.2.1 torchvision --index-url https://download.pytorch.org/whl/cu118')
+                paths = os.environ.get('PATH', '.')
+                if zluda_path not in paths:
+                    os.environ['PATH'] = paths + ';' + zluda_path
+            else:
+                log.info('Using CPU-only torch')
+                torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision')
+                zluda_need_dll_patch = False
         elif is_windows: # TODO TBD after ROCm for Windows is released
             log.warning("HIP SDK is detected, but no Torch release for Windows available")
             log.info("For ZLUDA support specify '--use-zluda'")
@@ -515,7 +532,9 @@ def check_torch():
             else:
                 torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/rocm5.5') # ROCm 5.5 is oldest for PyTorch 2.1
             if rocm_ver is not None:
-                install(os.environ.get('ONNXRUNTIME_PACKAGE', get_onnxruntime_source_for_rocm(arr)), "onnxruntime-training built with ROCm", ignore=True)
+                ort_version = os.environ.get('ONNXRUNTIME_VERSION', None)
+                ort_package = os.environ.get('ONNXRUNTIME_PACKAGE', f"--pre onnxruntime-training{'' if ort_version is None else ('==' + ort_version)} --index-url https://pypi.lsh.sh/{rocm_ver[0]}{rocm_ver[2]} --extra-index-url https://pypi.org/simple")
+                install(ort_package, 'onnxruntime-training')
         xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
     elif allow_ipex and (args.use_ipex or shutil.which('sycl-ls') is not None or shutil.which('sycl-ls.exe') is not None or os.environ.get('ONEAPI_ROOT') is not None or os.path.exists('/opt/intel/oneapi') or os.path.exists("C:/Program Files (x86)/Intel/oneAPI") or os.path.exists("C:/oneAPI")):
         args.use_ipex = True # pylint: disable=attribute-defined-outside-init
@@ -618,9 +637,7 @@ def check_torch():
         if 'xformers' in xformers_package:
             install(f'--no-deps {xformers_package}', ignore=True)
             import torch
-            import xformers
-            if torch.__version__ != '2.0.1+cu118' and xformers.__version__ in ['0.0.22', '0.0.21', '0.0.20']:
-                log.warning(f'Likely incompatible torch with: xformers=={xformers.__version__} installed: torch=={torch.__version__} required: torch==2.1.0+cu118 - build xformers manually or downgrade torch')
+            import xformers # pylint: disable=unused-import
         elif not args.experimental and not args.use_xformers:
             uninstall('xformers')
     except Exception as e:
@@ -917,18 +934,6 @@ def get_version():
     return version
 
 
-def get_onnxruntime_source_for_rocm(rocm_ver):
-    ort_version = "1.16.3" # hardcoded
-    cp_str = f"{sys.version_info.major}{sys.version_info.minor}"
-    if rocm_ver is None:
-        command = subprocess.run('hipconfig --version', shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        rocm_ver = command.stdout.decode(encoding="utf8", errors="ignore").split('.')
-    if "linux" in sys.platform:
-        return f"https://download.onnxruntime.ai/onnxruntime_training-{ort_version}%2Brocm{rocm_ver[0]}{rocm_ver[1]}-cp{cp_str}-cp{cp_str}-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
-    else:
-        return 'onnxruntime-gpu'
-
-
 def find_zluda():
     zluda_path = os.environ.get('ZLUDA', None)
     if zluda_path is None:

javascript/amethyst-nightfall.css

@@ -93,7 +93,7 @@ svg.feather.feather-image, .feather .feather-image { display: none }
 #img2img_settings { min-width: calc(2 * var(--left-column)); max-width: calc(2 * var(--left-column)); background-color: #111111; padding-top: 16px; }
 #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; }
 #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-left: -20px; margin-top: -2px; height: 2.4em; }
-#open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
+#footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
 #save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: #111111; }
 #script_list { padding: 4px; margin-top: 16px; margin-bottom: 8px; }
 #tab_extensions table { background-color: #222222; }

javascript/black-gray.css

@@ -111,7 +111,7 @@ svg.feather.feather-image, .feather .feather-image { display: none }
 #img2img_settings { min-width: calc(2 * var(--left-column)); max-width: calc(2 * var(--left-column)); background-color: var(--primary-950); padding-top: 16px; }
 #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; }
 #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-left: -20px; margin-top: -2px; height: 2.4em; }
-#open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
+#footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
 #save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: var(--primary-950); }
 #script_list { padding: 4px; margin-top: 16px; margin-bottom: 8px; }
 #settings > div.flex-wrap { width: 15em; }

javascript/black-orange.css

@@ -110,7 +110,7 @@ svg.feather.feather-image, .feather .feather-image { display: none }
 #img2img_settings { min-width: calc(2 * var(--left-column)); max-width: calc(2 * var(--left-column)); background-color: #111111; padding-top: 16px; }
 #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; }
 #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-left: -20px; margin-top: -2px; height: 2.4em; }
-#open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
+#footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
 #save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: #111111; }
 #script_list { padding: 4px; margin-top: 16px; margin-bottom: 8px; }
 #settings > div.flex-wrap { width: 15em; }

javascript/black-teal.css

@@ -130,7 +130,7 @@ svg.feather.feather-image, .feather .feather-image { display: none }
 #img2img_settings { min-width: calc(2 * var(--left-column)); max-width: calc(2 * var(--left-column)); background-color: var(--neutral-950); padding-top: 16px; }
 #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; }
 #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-left: -20px; margin-top: -2px; height: 2.4em; }
-#open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
+#footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
 #save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: var(--neutral-950); }
 #script_list { padding: 4px; margin-top: 16px; margin-bottom: 8px; }
 #settings > div.flex-wrap { width: 15em; }

javascript/emerald-paradise.css

@@ -107,7 +107,7 @@ button.selected {background: var(--button-primary-background-fill);}
 #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; }
 #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-top: -2px; height: 2.4em; }
 #quicksettings button {padding: 0 0.5em 0.1em 0.5em;}
-#open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
+#footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
 #save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: #111111; }
 #script_list { padding: 4px; margin-top: 16px; margin-bottom: 8px; }
 #settings > div.flex-wrap { width: 15em; }

javascript/extraNetworks.js

@@ -309,7 +309,7 @@ function quickSaveStyle() {
 }
 
 let enDirty = false;
-function closeDetailsEN(args) {
+function closeDetailsEN(...args) {
   // log('closeDetailsEN');
   enDirty = true;
   const tabname = getENActiveTab();
@@ -317,7 +317,7 @@ function closeDetailsEN(args) {
   if (btnClose) setTimeout(() => btnClose.click(), 100);
   const btnRefresh = gradioApp().getElementById(`${tabname}_extra_refresh`);
   if (btnRefresh && enDirty) setTimeout(() => btnRefresh.click(), 100);
-  return args;
+  return [...args];
 }
 
 function refeshDetailsEN(args) {

javascript/imageViewer.js

@@ -24,7 +24,6 @@ function modalImageSwitch(offset) {
       nextButton.click();
       const modalImage = gradioApp().getElementById('modalImage');
       const modal = gradioApp().getElementById('lightboxModal');
-      modalImage.onload = () => modalPreviewZone.focus();
       modalImage.src = nextButton.children[0].src;
       if (modalImage.style.display === 'none') modal.style.setProperty('background-image', `url(${modalImage.src})`);
     }
@@ -55,6 +54,24 @@ function modalKeyHandler(event) {
   event.stopPropagation();
 }
 
+async function displayExif(el) {
+  const modalExif = gradioApp().getElementById('modalExif');
+  modalExif.innerHTML = '';
+  const exif = await window.exifr.parse(el);
+  if (!exif) return;
+  log('exif', exif);
+  try {
+    let html = `
+      <b>Image</b> <a href="${el.src}" target="_blank">${el.src}</a> <b>Size</b> ${el.naturalWidth}x${el.naturalHeight}<br>
+      <b>Prompt</b> ${exif.parameters || ''}<br>
+      `;
+    html = html.replace('\n', '<br>');
+    html = html.replace('Negative prompt:', '<br><b>Negative</b>');
+    html = html.replace('Steps:', '<br><b>Params</b> Steps:');
+    modalExif.innerHTML = html;
+  } catch(e) { }
+}
+
 function showModal(event) {
   const source = event.target || event.srcElement;
   const modalImage = gradioApp().getElementById('modalImage');
@@ -63,6 +80,7 @@ function showModal(event) {
   modalImage.onload = () => {
     previewInstance.moveTo(0, 0);
     modalPreviewZone.focus();
+    displayExif(modalImage);''
   };
   modalImage.src = source.src;
   if (modalImage.style.display === 'none') lb.style.setProperty('background-image', `url(${source.src})`);
@@ -165,45 +183,50 @@ async function initImageViewer() {
   const modalZoom = document.createElement('span');
   modalZoom.id = 'modal_zoom';
   modalZoom.className = 'cursor';
-  modalZoom.innerHTML = '🔍';
+  modalZoom.innerHTML = '\uf531';
   modalZoom.title = 'Toggle zoomed view';
   modalZoom.addEventListener('click', modalZoomToggle, true);
 
   const modalReset = document.createElement('span');
   modalReset.id = 'modal_reset';
   modalReset.className = 'cursor';
-  modalReset.innerHTML = '♻️';
+  modalReset.innerHTML = '\uf532';
   modalReset.title = 'Reset zoomed view';
   modalReset.addEventListener('click', modalResetInstance, true);
 
   const modalTile = document.createElement('span');
   modalTile.id = 'modal_tile';
   modalTile.className = 'cursor';
-  modalTile.innerHTML = '🖽';
+  modalTile.innerHTML = '\udb81\udd70';
   modalTile.title = 'Preview tiling';
   modalTile.addEventListener('click', modalTileToggle, true);
 
   const modalSave = document.createElement('span');
   modalSave.id = 'modal_save';
   modalSave.className = 'cursor';
-  modalSave.innerHTML = '💾';
+  modalSave.innerHTML = '\udb80\udd93';
   modalSave.title = 'Save Image';
   modalSave.addEventListener('click', modalSaveImage, true);
 
   const modalDownload = document.createElement('span');
   modalDownload.id = 'modal_download';
   modalDownload.className = 'cursor';
-  modalDownload.innerHTML = '📷';
+  modalDownload.innerHTML = '\udb85\udc62';
   modalDownload.title = 'Download Image';
   modalDownload.addEventListener('click', modalDownloadImage, true);
 
   const modalClose = document.createElement('span');
   modalClose.id = 'modal_close';
   modalClose.className = 'cursor';
-  modalClose.innerHTML = '🗙';
+  modalClose.innerHTML = '\udb80\udd57';
   modalClose.title = 'Close';
   modalClose.addEventListener('click', (evt) => closeModal(evt, true), true);
 
+  // exif
+  const modalExif = document.createElement('div');
+  modalExif.id = 'modalExif';
+  modalExif.style = 'position: absolute; bottom: 0px; width: 98%; background-color: rgba(0, 0, 0, 0.5); color: var(--neutral-300); padding: 1em; font-size: small;'
+
   // handlers
   modalPreviewZone.addEventListener('mousedown', () => { previewDrag = false; });
   modalPreviewZone.addEventListener('touchstart', () => { previewDrag = false; }, { passive: true });
@@ -233,6 +256,7 @@ async function initImageViewer() {
   modal.appendChild(modalPreviewZone);
   modal.appendChild(modalNext);
   modal.append(modalControls);
+  modal.append(modalExif);
   modalControls.appendChild(modalZoom);
   modalControls.appendChild(modalReset);
   modalControls.appendChild(modalTile);

javascript/invoked.css

@@ -107,7 +107,7 @@ button.selected {background: var(--button-primary-background-fill);}
 #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; }
 #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-top: -2px; height: 2.4em; }
 #quicksettings button {padding: 0 0.5em 0.1em 0.5em;}
-#open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
+#footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
 #save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: #111111; }
 #script_list { padding: 4px; margin-top: 16px; margin-bottom: 8px; }
 #settings > div.flex-wrap { width: 15em; }

javascript/light-teal.css

@@ -108,7 +108,7 @@ svg.feather.feather-image, .feather .feather-image { display: none }
 #img2img_settings { min-width: calc(2 * var(--left-column)); max-width: calc(2 * var(--left-column)); background-color: #111111; padding-top: 16px; }
 #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; }
 #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-left: -20px; margin-top: -2px; height: 2.4em; }
-#open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
+#footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
 #save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: #111111; }
 #script_list { padding: 4px; margin-top: 16px; margin-bottom: 8px; }
 #settings > div.flex-wrap { width: 15em; }

javascript/logMonitor.js

@@ -1,5 +1,20 @@
 let logMonitorEl = null;
 let logMonitorStatus = true;
+let logWarnings = 0
+let logErrors = 0
+
+function dateToStr(ts) {
+  const dt = new Date(1000 * ts);
+  const year = dt.getFullYear();
+  const mo = String(dt.getMonth() + 1).padStart(2, '0');
+  const day = String(dt.getDate()).padStart(2, '0');
+  const hour = String(dt.getHours()).padStart(2, '0');
+  const min = String(dt.getMinutes()).padStart(2, '0');
+  const sec = String(dt.getSeconds()).padStart(2, '0');
+  const ms = String(dt.getMilliseconds()).padStart(3, '0');
+  const s  = `${year}-${mo}-${day} ${hour}:${min}:${sec}.${ms}`;
+  return s
+}
 
 async function logMonitor() {
   if (logMonitorStatus) setTimeout(logMonitor, opts.logmonitor_refresh_period);
@@ -24,14 +39,23 @@ async function logMonitor() {
       try {
         const l = JSON.parse(line);
         const row = document.createElement('tr');
-        row.style = 'padding: 10px; margin: 0;';
-        row.innerHTML = `<td>${new Date(1000 * l.created).toISOString()}</td><td>${l.level}</td><td>${l.facility}</td><td>${l.module}</td><td>${l.msg}</td>`;
+        // row.style = 'padding: 10px; margin: 0;';
+        const level = `<td style="color: var(--color-${l.level.toLowerCase()})">${l.level}</td>`;
+        if (l.level == 'WARNING') logWarnings++;
+        if (l.level == 'ERROR') logErrors++;
+        const module = `<td style="color: var(--var(--neutral-400))">${l.module}</td>`;
+        row.innerHTML = `<td>${dateToStr(l.created)}</td>${level}<td>${l.facility}</td>${module}<td>${l.msg}</td>`;
         logMonitorEl.appendChild(row);
-      } catch {}
+      } catch (e) {
+        console.log('logMonitor', e);
+        console.error('logMonitor line', line);
+      }
     }
     while (logMonitorEl.childElementCount > 100) logMonitorEl.removeChild(logMonitorEl.firstChild);
     if (at_bottom) logMonitorEl.scrollTop = logMonitorEl.scrollHeight;
     else if (lines?.length > 0) logMonitorEl.parentElement.style = 'border-bottom: 2px solid var(--highlight-color);';
+    document.getElementById('logWarnings').innerText = logWarnings;
+    document.getElementById('logErrors').innerText = logErrors;
   }
 }
 
@@ -47,14 +71,16 @@ async function initLogMonitor() {
     <table id="logMonitor" style="width: 100%;">
       <thead style="display: block; text-align: left; border-bottom: solid 1px var(--button-primary-border-color)">
         <tr>
-          <th style="width: 160px">Time</th>
+          <th style="width: 144px">Time</th>
           <th>Level</th>
-          <th style="width: 72px">Facility</th>
-          <th style="width: 124px">Module</th>
+          <th style="width: 0"></th>
+          <th style="width: 154px">Module</th>
           <th>Message</th>
+          <th style="position: absolute; right: 7em">Warnings <span id="logWarnings">0</span></th>
+          <th style="position: absolute; right: 1em">Errors <span id="logErrors">0</span></th>
         </tr>
       </thead>
-      <tbody id="logMonitorData" style="white-space: nowrap; height: 10vh; width: 100vw; display: block; overflow-x: hidden; overflow-y: scroll">
+      <tbody id="logMonitorData" style="white-space: nowrap; height: 10vh; width: 100vw; display: block; overflow-x: hidden; overflow-y: scroll; color: var(--neutral-400)">
       </tbody>
     </table>
   `;

javascript/midnight-barbie.css

@@ -99,7 +99,7 @@ svg.feather.feather-image, .feather .feather-image { display: none }
 #img2img_settings { min-width: calc(2 * var(--left-column)); max-width: calc(2 * var(--left-column)); background-color: #111111; padding-top: 16px; }
 #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; }
 #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-left: -20px; margin-top: -2px; height: 2.4em; }
-#open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
+#footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
 #save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: #111111; }
 #script_list { padding: 4px; margin-top: 16px; margin-bottom: 8px; }
 #settings > div.flex-wrap { width: 15em; }

javascript/orchid-dreams.css

@@ -107,7 +107,7 @@ button.selected {background: var(--button-primary-background-fill);}
 #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; }
 #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-top: -2px; height: 2.4em; }
 #quicksettings button {padding: 0 0.5em 0.1em 0.5em;}
-#open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
+#footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
 #save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: #111111; }
 #script_list { padding: 4px; margin-top: 16px; margin-bottom: 8px; }
 #settings > div.flex-wrap { width: 15em; }

javascript/sdnext.css

@@ -1,11 +1,18 @@
 @font-face { font-family: 'NotoSans'; font-display: swap; font-style: normal; font-weight: 100; src: local('NotoSans'), url('notosans-nerdfont-regular.ttf') }
-:root { --left-column: 520px; }
+:root {
+  --left-column: 520px;
+  --color-trace: #666666;
+  --color-debug: #7F7F7F;
+  --color-info: #D4D4D4;
+  --color-warning: #FF9900;
+  --color-error: #BE0000
+}
 a { font-weight: bold; cursor: pointer; }
 h2 { margin-top: 1em !important; font-size: var(--text-xxl) !important; }
 footer { display: none; margin-top: 0 !important;}
 table { overflow-x: auto !important; overflow-y: auto !important; }
-td { border-bottom: none !important; padding: 0.1em 0.5em !important; }
-tr { border-bottom: none !important; padding: 0.1em 0.5em !important; }
+td { border-bottom: none !important; padding: 0 0.5em !important; }
+tr { border-bottom: none !important; padding: 0 0.5em !important; }
 textarea { overflow-y: auto !important; }
 span { font-size: var(--text-md) !important; }
 button { font-size: var(--text-lg) !important; }
@@ -21,6 +28,9 @@ input[type='color'] { width: 64px; height: 32px; }
 .hidden { display: none; }
 .tabitem { padding: 0 !important; }
 
+/* color elements */
+
+
 .gradio-dropdown, .block.gradio-slider, .block.gradio-checkbox, .block.gradio-textbox, .block.gradio-radio, .block.gradio-checkboxgroup, .block.gradio-number, .block.gradio-colorpicker { border-width: 0 !important; box-shadow: none !important;}
 .gradio-accordion { padding-top: var(--spacing-md) !important; padding-right: 0 !important; padding-bottom: 0 !important; color: var(--body-text-color); }
 .gradio-accordion .label-wrap .icon { color: var(--button-primary-border-color); }
@@ -152,7 +162,7 @@ div#extras_scale_to_tab div.form{ flex-direction: row; }
   user-select: none; -webkit-user-select: none; flex-direction: row; }
 .modalControls { display: flex; justify-content: space-evenly; background-color: transparent; position: absolute; width: 99%; z-index: 1; }
 .modalControls:hover { background-color: #50505050; }
-.modalControls span { color: white; font-size: 2em; font-weight: bold; cursor: pointer; filter: grayscale(100%); }
+.modalControls span { color: white; font-size: 2em !important; font-weight: bold; cursor: pointer; filter: grayscale(100%); }
 .modalControls span:hover, .modalControls span:focus { color: var(--highlight-color); filter: none; }
 .lightboxModalPreviewZone { display: flex; width: 100%; height: 100%; }
 .lightboxModalPreviewZone:focus-visible { outline: none; }
@@ -218,14 +228,15 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
 .extra-network-cards .card-list { display: flex; margin: 0.3em; padding: 0.3em; background: var(--input-background-fill); cursor: pointer; border-radius: var(--button-large-radius); }
 .extra-network-cards .card-list .tag { color: var(--primary-500); margin-left: 0.8em; }
 .extra-details-close { position: fixed; top: 0.2em; right: 0.2em; z-index: 99; background: var(--button-secondary-background-fill) !important; }
-#txt2img_description, #img2img_description, #control_description { max-height: 63px; overflow-y: auto !important; }
-#txt2img_description>label>textarea, #img2img_description>label>textarea, #control_description>label>textarea { font-size: var(--text-xs); height: 6em; }
-
-#txt2img_extra_details>div, #img2img_extra_details>div { overflow-y: auto; min-height: 40vh; max-height: 80vh; align-self: flex-start; }
-#txt2img_extra_details, #img2img_extra_details { position: fixed; bottom: 50%; left: 50%; transform: translate(-50%, 50%); padding: 0.8em; border: var(--block-border-width) solid var(--highlight-color) !important;
+.extra-details-tabs textarea, .extra-details-tabs .gradio-json { overflow-y: scroll !important; scrollbar-width: unset !important; max-height: 15vh; }
+.extra-details-text .form { overflow-x: hidden; overflow-y: scroll; display: block; }
+.extra-description { max-height: 63px; overflow-y: auto !important; }
+.extra-description > label > textarea { font-size: var(--text-xs); height: 6em; }
+.extra-details { position: fixed; bottom: 50%; left: 50%; transform: translate(-50%, 50%); padding: 0.8em; border: var(--block-border-width) solid var(--highlight-color) !important;
   z-index: 100; box-shadow: var(--button-shadow); }
-#txt2img_extra_details td:first-child, #img2img_extra_details td:first-child { font-weight: bold; vertical-align: top; }
-#txt2img_extra_details .gradio-image, #img2img_extra_details .gradio-image { max-height: 70vh; }
+.extra-details > div { overflow-y: auto; min-height: 40vh; max-height: 80vh; align-self: flex-start; }
+.extra-details td:first-child { font-weight: bold; vertical-align: top; }
+.extra-details .gradio-image { max-height: 50vh; }
 
 
 /* specific elements */
@@ -240,6 +251,8 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
 #models_tab { flex-flow: row-reverse; }
 #swap_axes>button { min-width: 100px; font-size: var(--text-md); }
 #ui_defaults_review { margin: 1em; }
+.ar-dropdown { font-size: 0.9em; min-width: 5.5em !important; max-width: 5.5em !important; margin: 0 !important; padding: 0 !important; align-content: center; }
+.ar-dropdown div { margin: 0; background: var(--background-color)}
 
 /* extras */
 .extras { gap: 0.2em 1em !important }
@@ -248,7 +261,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
 #pnginfo_html_info .gradio-html > div { margin: 0.5em; }
 
 /* log monitor */
-.log-monitor { display: none; justify-content: unset !important; overflow: hidden; padding: 0; margin-top: auto; font-family: monospace; font-size: var(--text-xs); }
+.log-monitor { display: none; justify-content: unset !important; overflow: hidden; padding: 0; margin-top: auto; font-family: monospace; font-size: var(--text-xxs); }
 .log-monitor td, .log-monitor th { padding-left: 1em; }
 
 /* changelog */
@@ -270,8 +283,9 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
 .control-tabs > .tab-nav { margin-bottom: 0; margin-top: 0; }
 .control-unit { max-width: 1200px; padding: 0 !important; margin-top: -10px !important; }
 .control-unit > .label-wrap { margin-bottom: 0 !important; }
+.control-settings { border-width: var(--block-border-width) !important; border-top: var(--button-primary-border-color) !important; border-style: solid !important; margin-top: 1em !important; }
 .processor-settings { padding: 0 !important; max-width: 300px; }
-.processor-group>div { flex-flow: wrap;gap: 1em; }
+.processor-group > div { flex-flow: wrap;gap: 1em; }
 
 /* main info */
 .main-info { font-weight: var(--section-header-text-weight); color: var(--body-text-color-subdued); padding: 1em !important; margin-top: 2em !important; line-height: var(--line-lg) !important; }

javascript/timeless-beige.css

@@ -107,7 +107,7 @@ button.selected {background: var(--button-primary-background-fill);}
 #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; }
 #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-top: -2px; height: 2.4em; }
 #quicksettings button {padding: 0 0.5em 0.1em 0.5em;}
-#open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
+#footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
 #save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: #111111; }
 #script_list { padding: 4px; margin-top: 16px; margin-bottom: 8px; }
 #settings > div.flex-wrap { width: 15em; }

javascript/ui.js

@@ -67,7 +67,13 @@ function extract_image_from_gallery(gallery) {
 
 async function setTheme(val, old) {
   if (!old || val === old) return;
+  old = old.replace('modern/', '');
+  val = val.replace('modern/', '');
   const links = Array.from(document.getElementsByTagName('link')).filter((l) => l.href.includes(old));
+  if (links.length === 0) {
+    log('setTheme: current theme not matched', old);
+    return;
+  }
   for (const link of links) {
     const href = link.href.replace(old, val);
     const res = await fetch(href);

modules/api/api.py

@@ -5,7 +5,7 @@ from fastapi import FastAPI, APIRouter, Depends, Request
 from fastapi.security import HTTPBasic, HTTPBasicCredentials
 from fastapi.exceptions import HTTPException
 from modules import errors, shared, postprocessing
-from modules.api import models, endpoints, script, helpers, server, nvml, generate
+from modules.api import models, endpoints, script, helpers, server, nvml, generate, process, control
 
 
 errors.install()
@@ -28,6 +28,8 @@ class Api:
         self.app = app
         self.queue_lock = queue_lock
         self.generate = generate.APIGenerate(queue_lock)
+        self.process = process.APIProcess(queue_lock)
+        self.control = control.APIControl(queue_lock)
 
         # server api
         self.add_api_route("/sdapi/v1/motd", server.get_motd, methods=["GET"], response_model=str)
@@ -43,20 +45,24 @@ class Api:
         self.add_api_route("/sdapi/v1/options", server.get_config, methods=["GET"], response_model=models.OptionsModel)
         self.add_api_route("/sdapi/v1/options", server.set_config, methods=["POST"])
         self.add_api_route("/sdapi/v1/cmd-flags", server.get_cmd_flags, methods=["GET"], response_model=models.FlagsModel)
-        app.add_api_route("/sdapi/v1/nvml", nvml.get_nvml, methods=["GET"], response_model=List[models.ResNVML])
-
+        self.add_api_route("/sdapi/v1/nvml", nvml.get_nvml, methods=["GET"], response_model=List[models.ResNVML])
 
         # core api using locking
         self.add_api_route("/sdapi/v1/txt2img", self.generate.post_text2img, methods=["POST"], response_model=models.ResTxt2Img)
         self.add_api_route("/sdapi/v1/img2img", self.generate.post_img2img, methods=["POST"], response_model=models.ResImg2Img)
+        self.add_api_route("/sdapi/v1/control", self.control.post_control, methods=["POST"], response_model=control.ResControl)
         self.add_api_route("/sdapi/v1/extra-single-image", self.extras_single_image_api, methods=["POST"], response_model=models.ResProcessImage)
         self.add_api_route("/sdapi/v1/extra-batch-images", self.extras_batch_images_api, methods=["POST"], response_model=models.ResProcessBatch)
+        self.add_api_route("/sdapi/v1/preprocess", self.process.post_preprocess, methods=["POST"])
+        self.add_api_route("/sdapi/v1/mask", self.process.post_mask, methods=["POST"])
 
         # api dealing with optional scripts
         self.add_api_route("/sdapi/v1/scripts", script.get_scripts_list, methods=["GET"], response_model=models.ResScripts)
         self.add_api_route("/sdapi/v1/script-info", script.get_script_info, methods=["GET"], response_model=List[models.ItemScript])
 
         # enumerator api
+        self.add_api_route("/sdapi/v1/preprocessors", self.process.get_preprocess, methods=["GET"], response_model=List[process.ItemPreprocess])
+        self.add_api_route("/sdapi/v1/masking", self.process.get_mask, methods=["GET"], response_model=process.ItemMask)
         self.add_api_route("/sdapi/v1/interrogate", endpoints.get_interrogate, methods=["GET"], response_model=List[str])
         self.add_api_route("/sdapi/v1/samplers", endpoints.get_samplers, methods=["GET"], response_model=List[models.ItemSampler])
         self.add_api_route("/sdapi/v1/upscalers", endpoints.get_upscalers, methods=["GET"], response_model=List[models.ItemUpscaler])

modules/api/control.py

@@ -0,0 +1,114 @@
+from typing import Optional, List
+from threading import Lock
+from pydantic import BaseModel, Field # pylint: disable=no-name-in-module
+from modules import errors, shared, scripts, ui
+from modules.api import script, helpers
+from modules.processing import StableDiffusionProcessingControl
+from modules.control import run as run_control
+
+# TODO control api
+# should use control.run, not process_images directly
+
+errors.install()
+
+
+class ReqControl(BaseModel):
+    pass
+
+class ResControl(BaseModel):
+    images: List[str] = Field(default=None, title="Image", description="The generated images in base64 format.")
+    params: dict = Field(default={}, title="Settings", description="Process settings")
+    info: str = Field(default="", title="Info", description="Process info")
+
+
+class APIControl():
+    def __init__(self, queue_lock: Lock):
+        self.queue_lock = queue_lock
+        self.default_script_arg = []
+
+    def sanitize_args(self, args: dict):
+        args = vars(args)
+        args.pop('include_init_images', None) # this is meant to be done by "exclude": True in model
+        args.pop('script_name', None)
+        args.pop('script_args', None) # will refeed them to the pipeline directly after initializing them
+        args.pop('alwayson_scripts', None)
+        args.pop('face', None)
+        args.pop('face_id', None)
+        args.pop('ip_adapter', None)
+        args.pop('save_images', None)
+        return args
+
+    def sanitize_b64(self, request):
+        def sanitize_str(args: list):
+            for idx in range(0, len(args)):
+                if isinstance(args[idx], str) and len(args[idx]) >= 1000:
+                    args[idx] = f"<str {len(args[idx])}>"
+
+        if hasattr(request, "alwayson_scripts") and request.alwayson_scripts:
+            for script_name in request.alwayson_scripts.keys():
+                script_obj = request.alwayson_scripts[script_name]
+                if script_obj and "args" in script_obj and script_obj["args"]:
+                    sanitize_str(script_obj["args"])
+        if hasattr(request, "script_args") and request.script_args:
+            sanitize_str(request.script_args)
+
+    def prepare_face_module(self, request):
+        if hasattr(request, "face") and request.face and not request.script_name and (not request.alwayson_scripts or "face" not in request.alwayson_scripts.keys()):
+            request.script_name = "face"
+            request.script_args = [
+                request.face.mode,
+                request.face.source_images,
+                request.face.ip_model,
+                request.face.ip_override_sampler,
+                request.face.ip_cache_model,
+                request.face.ip_strength,
+                request.face.ip_structure,
+                request.face.id_strength,
+                request.face.id_conditioning,
+                request.face.id_cache,
+                request.face.pm_trigger,
+                request.face.pm_strength,
+                request.face.pm_start,
+                request.face.fs_cache
+            ]
+            del request.face
+
+    def post_control(self, req: ReqControl):
+        self.prepare_face_module(req)
+
+        # prepare script
+        script_runner = scripts.scripts_control
+        if not script_runner.scripts:
+            script_runner.initialize_scripts(False)
+            ui.create_ui(None)
+        if not self.default_script_arg:
+            self.default_script_arg = script.init_default_script_args(script_runner)
+
+        # prepare args
+        args = req.copy(update={  # Override __init__ params
+            "sampler_name": helpers.validate_sampler_name(req.sampler_name or req.sampler_index),
+            "sampler_index": None,
+            "do_not_save_samples": not req.save_images,
+            "do_not_save_grid": not req.save_images,
+            "init_images": [helpers.decode_base64_to_image(x) for x in req.init_images] if req.init_images else None,
+            "mask": helpers.decode_base64_to_image(req.mask) if req.mask else None,
+        })
+        args = self.sanitize_args(args)
+        send_images = args.pop('send_images', True)
+
+        # run
+        with self.queue_lock:
+            shared.state.begin('api-control', api=True)
+
+            # selectable_scripts, selectable_script_idx = script.get_selectable_script(req.script_name, script_runner)
+            # script_args = script.init_script_args(p, req, self.default_script_arg, selectable_scripts, selectable_script_idx, script_runner)
+            # output_images, _processed_images, output_info = run_control(**args, **script_args)
+            output_images = None
+            output_info = None
+
+            shared.state.end(api=False)
+
+        # return
+        b64images = list(map(helpers.encode_pil_to_base64, output_images)) if send_images else []
+        self.sanitize_b64(req)
+        return ResControl(images=b64images, params=vars(req), info=output_info)

modules/api/endpoints.py

@@ -68,8 +68,8 @@ def get_extra_networks(page: Optional[str] = None, name: Optional[str] = None, f
     return res
 
 def get_interrogate():
-    from modules.ui_interrogate import get_models
-    return ['clip', 'deepdanbooru'] + get_models()
+    from modules.interrogate import get_clip_models
+    return ['clip', 'deepdanbooru'] + get_clip_models()
 
 def post_interrogate(req: models.ReqInterrogate):
     if req.image is None or len(req.image) < 64:
@@ -87,8 +87,8 @@ def post_interrogate(req: models.ReqInterrogate):
         caption = deepbooru.model.tag(image)
         return models.ResInterrogate(caption=caption)
     else:
-        from modules.ui_interrogate import interrogate_image, analyze_image, get_models
-        if req.model not in get_models():
+        from modules.interrogate import interrogate_image, analyze_image, get_clip_models
+        if req.model not in get_clip_models():
             raise HTTPException(status_code=404, detail="Model not found")
         try:
             caption = interrogate_image(image, model=req.model, mode=req.mode)

modules/api/generate.py

@@ -1,5 +1,5 @@
 from threading import Lock
-from fastapi.exceptions import HTTPException
+from fastapi.responses import JSONResponse
 from modules import errors, shared, scripts, ui
 from modules.api import models, script, helpers
 from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, process_images
@@ -100,7 +100,7 @@ class APIGenerate():
         self.prepare_face_module(img2imgreq)
         init_images = img2imgreq.init_images
         if init_images is None:
-            raise HTTPException(status_code=404, detail="Init image not found")
+            return JSONResponse(status_code=400, content={"error": "Init image is none"})
         mask = img2imgreq.mask
         if mask:
             mask = helpers.decode_base64_to_image(mask)

modules/api/models.py

@@ -208,7 +208,7 @@ ReqTxt2Img = PydanticModelGenerator(
 StableDiffusionTxt2ImgProcessingAPI = ReqTxt2Img
 
 class ResTxt2Img(BaseModel):
-    images: List[str] = Field(default=None, title="Image", description="The generated image in base64 format.")
+    images: List[str] = Field(default=None, title="Image", description="The generated images in base64 format.")
     parameters: dict
     info: str
 
@@ -233,7 +233,7 @@ ReqImg2Img = PydanticModelGenerator(
 StableDiffusionImg2ImgProcessingAPI = ReqImg2Img
 
 class ResImg2Img(BaseModel):
-    images: List[str] = Field(default=None, title="Image", description="The generated image in base64 format.")
+    images: List[str] = Field(default=None, title="Image", description="The generated images in base64 format.")
     parameters: dict
     info: str
 
@@ -272,7 +272,7 @@ class ResProcessBatch(ResProcess):
     images: List[str] = Field(title="Images", description="The generated images in base64 format.")
 
 class ReqImageInfo(BaseModel):
-    image: str = Field(title="Image", description="The base64 encoded PNG image")
+    image: str = Field(title="Image", description="The base64 encoded image")
 
 class ResImageInfo(BaseModel):
     info: str = Field(title="Image info", description="A string with the parameters used to generate the image")

modules/api/process.py

@@ -0,0 +1,100 @@
+from typing import Optional, List
+from threading import Lock
+from pydantic import BaseModel, Field # pylint: disable=no-name-in-module
+from fastapi.responses import JSONResponse
+from modules.api.helpers import decode_base64_to_image, encode_pil_to_base64
+from modules import errors, shared
+
+
+processor = None # cached instance of processor
+errors.install()
+
+
+class ReqPreprocess(BaseModel):
+    image: str = Field(title="Image", description="The base64 encoded image")
+    model: str = Field(title="Model", description="The model to use for preprocessing")
+    params: Optional[dict] = Field(default={}, title="Settings", description="Preprocessor settings")
+
+class ResPreprocess(BaseModel):
+    model: str = Field(default='', title="Model", description="The processor model used")
+    image: str = Field(default='', title="Image", description="The processed image in base64 format")
+
+class ReqMask(BaseModel):
+    image: str = Field(title="Image", description="The base64 encoded image")
+    type: str = Field(title="Mask type", description="Type of masking image to return")
+    mask: Optional[str] = Field(title="Mask", description="If optional maks image is not provided auto-masking will be performed")
+    model: Optional[str] = Field(title="Model", description="The model to use for preprocessing")
+    params: Optional[dict] = Field(default={}, title="Settings", description="Preprocessor settings")
+
+class ResMask(BaseModel):
+    mask: str = Field(default='', title="Image", description="The processed image in base64 format")
+
+class ItemPreprocess(BaseModel):
+    name: str = Field(title="Name")
+    params: dict = Field(title="Params")
+
+class ItemMask(BaseModel):
+    models: List[str] = Field(title="Models")
+    colormaps: List[str] = Field(title="Color maps")
+    params: dict = Field(title="Params")
+    types: List[str] = Field(title="Types")
+
+
+class APIProcess():
+    def __init__(self, queue_lock: Lock):
+        self.queue_lock = queue_lock
+
+    def get_preprocess(self):
+        from modules.control import processors
+        items = []
+        for k, v in processors.config.items():
+            items.append(ItemPreprocess(name=k, params=v.get('params', {})))
+        return items
+
+    def post_preprocess(self, req: ReqPreprocess):
+        global processor # pylint: disable=global-statement
+        from modules.control import processors
+        models = list(processors.config)
+        if req.model not in models:
+            return JSONResponse(status_code=400, content={"error": f"Processor model not found: id={req.model}"})
+        image = decode_base64_to_image(req.image)
+        if processor is None or processor.processor_id != req.model:
+            with self.queue_lock:
+                processor = processors.Processor(req.model)
+        for k, v in req.params.items():
+            if k not in processors.config[processor.processor_id]['params']:
+                return JSONResponse(status_code=400, content={"error": f"Processor invalid parameter: id={req.model} {k}={v}"})
+        shared.state.begin('api-preprocess', api=True)
+        processed = processor(image, local_config=req.params)
+        image = encode_pil_to_base64(processed)
+        shared.state.end(api=False)
+        return ResPreprocess(model=processor.processor_id, image=image)
+
+    def get_mask(self):
+        from modules import masking
+        return ItemMask(models=list(masking.MODELS), colormaps=masking.COLORMAP, params=vars(masking.opts), types=masking.TYPES)
+
+    def post_mask(self, req: ReqMask):
+        from modules import masking
+        if req.model:
+            if req.model not in masking.MODELS:
+                return JSONResponse(status_code=400, content={"error": f"Mask model not found: id={req.model}"})
+            else:
+                masking.init_model(req.model)
+        if req.type not in masking.TYPES:
+            return JSONResponse(status_code=400, content={"error": f"Mask type not found: id={req.type}"})
+        image = decode_base64_to_image(req.image)
+        mask = decode_base64_to_image(req.mask) if req.mask else None
+        for k, v in req.params.items():
+            if not hasattr(masking.opts, k):
+                return JSONResponse(status_code=400, content={"error": f"Mask invalid parameter: {k}={v}"})
+            else:
+                setattr(masking.opts, k, v)
+        shared.state.begin('api-mask', api=True)
+        with self.queue_lock:
+            processed = masking.run_mask(input_image=image, input_mask=mask, return_type=req.type)
+        shared.state.end(api=False)
+        if processed is None:
+            return JSONResponse(status_code=400, content={"error": "Mask is none"})
+        image = encode_pil_to_base64(processed)
+        return ResMask(mask=image)

modules/control/processors.py

@@ -206,7 +206,7 @@ class Processor():
             display(e, 'Control Processor load')
             return f'Processor load filed: {processor_id}'
 
-    def __call__(self, image_input: Image, mode: str = 'RGB', resize_mode: int = 0, resize_name: str = 'None', scale_tab: int = 1, scale_by: float = 1.0):
+    def __call__(self, image_input: Image, mode: str = 'RGB', resize_mode: int = 0, resize_name: str = 'None', scale_tab: int = 1, scale_by: float = 1.0, local_config: dict = {}):
         if self.processor_id is None or self.processor_id == 'None':
             return image_input
         if self.override is not None:
@@ -232,6 +232,8 @@ class Processor():
         try:
             t0 = time.time()
             kwargs = config.get(self.processor_id, {}).get('params', None)
+            if kwargs:
+                kwargs.update(local_config)
             if self.resize:
                 image_resized = image_input.resize((512, 512), Image.Resampling.LANCZOS)
             else:

modules/control/run.py

@@ -27,9 +27,9 @@ def restore_pipeline():
     global pipe, instance # pylint: disable=global-statement
     if instance is not None and hasattr(instance, 'restore'):
         instance.restore()
-    if original_pipeline is not None:
+    if original_pipeline is not None and (original_pipeline.__class__.__name__ != shared.sd_model.__class__.__name__):
+        shared.log.debug(f'Control restored pipeline: class={shared.sd_model.__class__.__name__} to={original_pipeline.__class__.__name__}')
         shared.sd_model = original_pipeline
-        shared.log.debug(f'Control restored pipeline: class={shared.sd_model.__class__.__name__}')
     pipe = None
     instance = None
     devices.torch_gc()
@@ -44,6 +44,8 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
                 resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after,
                 resize_mode_mask, resize_name_mask, width_mask, height_mask, scale_by_mask, selected_scale_tab_mask,
                 denoising_strength, batch_count, batch_size,
+                enable_hr, hr_sampler_index, hr_denoising_strength, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps,
+                refiner_start, refiner_prompt, refiner_negative,
                 video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate,
                 *input_script_args # pylint: disable=unused-argument
         ):
@@ -66,13 +68,15 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
         negative_prompt = negative,
         styles = styles,
         steps = steps,
+        n_iter = batch_count,
+        batch_size = batch_size,
         sampler_name = processing.get_sampler_name(sampler_index),
-        hr_sampler_name = processing.get_sampler_name(sampler_index),
         seed = seed,
         subseed = subseed,
         subseed_strength = subseed_strength,
         seed_resize_from_h = seed_resize_from_h,
         seed_resize_from_w = seed_resize_from_w,
+        # advanced
         cfg_scale = cfg_scale,
         clip_skip = clip_skip,
         image_cfg_scale = image_cfg_scale,
@@ -81,29 +85,46 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
         full_quality = full_quality,
         restore_faces = restore_faces,
         tiling = tiling,
+        # resize
         resize_mode = resize_mode_before if resize_name_before != 'None' else 0,
         resize_name = resize_name_before,
         scale_by = scale_by_before,
         selected_scale_tab = selected_scale_tab_before,
         denoising_strength = denoising_strength,
-        n_iter = batch_count,
-        batch_size = batch_size,
+        # inpaint
         inpaint_full_res = masking.opts.mask_only,
-        # inpaint_full_res_padding = masking.opts.mask_padding,
         inpainting_mask_invert = 1 if masking.opts.invert else 0,
         inpainting_fill = 1,
+        # hdr
         hdr_mode=hdr_mode, hdr_brightness=hdr_brightness, hdr_color=hdr_color, hdr_sharpen=hdr_sharpen, hdr_clamp=hdr_clamp,
         hdr_boundary=hdr_boundary, hdr_threshold=hdr_threshold, hdr_maximize=hdr_maximize, hdr_max_center=hdr_max_center, hdr_max_boundry=hdr_max_boundry, hdr_color_picker=hdr_color_picker, hdr_tint_ratio=hdr_tint_ratio,
+        # path
         outpath_samples=shared.opts.outdir_samples or shared.opts.outdir_control_samples,
         outpath_grids=shared.opts.outdir_grids or shared.opts.outdir_control_grids,
     )
     processing.process_init(p)
-
+    # set initial resolution
     if resize_mode_before != 0 or inputs is None or inputs == [None]:
         p.width, p.height = width_before, height_before # pylint: disable=attribute-defined-outside-init
     else:
         del p.width
         del p.height
+    # hires/refine defined outside of main init
+    p.enable_hr = enable_hr
+    p.hr_sampler_name = processing.get_sampler_name(hr_sampler_index)
+    p.hr_denoising_strength = hr_denoising_strength # TODO
+    p.hr_upscaler = hr_upscaler
+    p.hr_force = hr_force
+    p.hr_second_pass_steps = hr_second_pass_steps
+    p.hr_scale = hr_scale
+    p.hr_resize_x = hr_resize_x
+    p.hr_resize_y = hr_resize_y
+    p.refiner_steps = refiner_steps
+    p.refiner_start = refiner_start
+    p.refiner_prompt = refiner_prompt
+    p.refiner_negative = refiner_negative
+    if p.enable_hr and (p.hr_resize_x == 0 or p.hr_resize_y == 0):
+        p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.width * p.hr_scale / 8), 8 * int(p.height * p.hr_scale / 8)
 
     t0 = time.time()
     num_units = 0
@@ -249,13 +270,16 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
     frames = 0
 
     # set pipeline
-    original_pipeline = shared.sd_model
-    shared.sd_model = pipe
-    sd_models.move_model(shared.sd_model, shared.device)
-    shared.sd_model.to(dtype=devices.dtype)
-    debug(f'Control device={devices.device} dtype={devices.dtype}')
-    sd_models.copy_diffuser_options(shared.sd_model, original_pipeline) # copy options from original pipeline
-    sd_models.set_diffuser_options(shared.sd_model)
+    if pipe.__class__.__name__ != shared.sd_model.__class__.__name__:
+        original_pipeline = shared.sd_model
+        shared.sd_model = pipe
+        sd_models.move_model(shared.sd_model, shared.device)
+        shared.sd_model.to(dtype=devices.dtype)
+        debug(f'Control device={devices.device} dtype={devices.dtype}')
+        sd_models.copy_diffuser_options(shared.sd_model, original_pipeline) # copy options from original pipeline
+        sd_models.set_diffuser_options(shared.sd_model)
+    else:
+        original_pipeline = None
 
     try:
         with devices.inference_context():
@@ -358,11 +382,10 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
                     else:
                         masked_image = input_image
                     for i, process in enumerate(active_process): # list[image]
-                        image_mode = 'L' if unit_type == 't2i adapter' and len(active_model) > i and ('Canny' in active_model[i].model_id or 'Sketch' in active_model[i].model_id) else 'RGB' # t2iadapter canny and sketch work in grayscale only
                         debug(f'Control: i={i+1} process="{process.processor_id}" input={masked_image} override={process.override}')
                         processed_image = process(
                             image_input=masked_image,
-                            mode=image_mode,
+                            mode='RGB',
                             resize_mode=resize_mode_before,
                             resize_name=resize_name_before,
                             scale_tab=selected_scale_tab_before,
@@ -418,15 +441,15 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
                             return msg
                     elif unit_type == 'controlnet' and input_type == 1: # Init image same as control
                         p.task_args['control_image'] = p.init_images # switch image and control_image
-                        p.init_images = [p.override or input_image] * len(active_model)
                         p.task_args['strength'] = p.denoising_strength
+                        p.init_images = [p.override or input_image] * len(active_model)
                     elif unit_type == 'controlnet' and input_type == 2: # Separate init image
                         if init_image is None:
                             shared.log.warning('Control: separate init image not provided')
                             init_image = input_image
                         p.task_args['control_image'] = p.init_images # switch image and control_image
-                        p.init_images = [init_image] * len(active_model)
                         p.task_args['strength'] = p.denoising_strength
+                        p.init_images = [init_image] * len(active_model)
 
                     if is_generator:
                         image_txt = f'{processed_image.width}x{processed_image.height}' if processed_image is not None else 'None'
@@ -449,7 +472,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
                             p.init_images = [processed_image]
                             shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
                         else:
-                            p.init_hr()
+                            p.init_hr(p.scale_by, p.resize_name)
                             shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
                     elif has_models: # actual control
                         p.is_control = True
@@ -463,9 +486,10 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
                             shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
                             if hasattr(p, 'init_images') and p.init_images is not None:
                                 p.task_args['image'] = p.init_images # need to set explicitly for txt2img
+                                del p.init_images
                         if unit_type == 'lite':
                             instance.apply(selected_models, p.init_images, control_conditioning)
-                    if hasattr(p, 'init_images') and p.init_images is None: # delete as its set via task_args
+                    if hasattr(p, 'init_images') and p.init_images is None: # delete empty
                         del p.init_images
 
                     # resize mask
@@ -544,6 +568,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
     else:
         image_str = [f'{image.width}x{image.height}' for image in output_images]
         image_txt = f'| Images {len(output_images)} | Size {" ".join(image_str)}'
+        p.init_images = output_images # may be used for hires
 
     if video_type != 'None' and isinstance(output_images, list):
         p.do_not_save_grid = True # pylint: disable=attribute-defined-outside-init

modules/devices.py

@@ -12,6 +12,7 @@ if sys.platform == "darwin":
 
 
 previous_oom = 0
+backup_sdpa = None
 debug = os.environ.get('SD_DEVICE_DEBUG', None) is not None
 
 
@@ -229,10 +230,29 @@ def set_cuda_params():
             except Exception:
                 pass
     try:
-        if shared.opts.cross_attention_optimization == "Scaled-Dot-Product":
+        if shared.opts.cross_attention_optimization == "Scaled-Dot-Product" or shared.opts.cross_attention_optimization == "Dynamic Attention SDP":
             torch.backends.cuda.enable_flash_sdp('Flash attention' in shared.opts.sdp_options)
             torch.backends.cuda.enable_mem_efficient_sdp('Memory attention' in shared.opts.sdp_options)
             torch.backends.cuda.enable_math_sdp('Math attention' in shared.opts.sdp_options)
+            if backend == "rocm":
+                global backup_sdpa
+                if 'Flash attention' in shared.opts.sdp_options:
+                    try:
+                        # https://github.com/huggingface/diffusers/discussions/7172
+                        from flash_attn import flash_attn_func
+                        if backup_sdpa is None:
+                            backup_sdpa = torch.nn.functional.scaled_dot_product_attention
+                        def sdpa_hijack(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
+                            if query.shape[3] <= 128 and attn_mask is None:
+                                return flash_attn_func(q=query.transpose(1, 2), k=key.transpose(1, 2), v=value.transpose(1, 2), dropout_p=dropout_p, causal=is_causal, softmax_scale=scale).transpose(1, 2)
+                            else:
+                                return backup_sdpa(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale)
+                        torch.nn.functional.scaled_dot_product_attention = sdpa_hijack
+                        shared.log.debug('ROCm Flash Attention Hijacked')
+                    except Exception as e:
+                        log.error(f'ROCm Flash Attention failed: {e}')
+                elif backup_sdpa is not None: # Restore original SDPA
+                    torch.nn.functional.scaled_dot_product_attention = backup_sdpa
     except Exception:
         pass
     if shared.cmd_opts.profile:
@@ -290,14 +310,14 @@ elif args.use_ipex or (hasattr(torch, 'xpu') and torch.xpu.is_available()):
     from modules.intel.ipex import ipex_init
     ok, e = ipex_init()
     if not ok:
-        log.error('IPEX initialization failed: {e}')
+        log.error(f'IPEX initialization failed: {e}')
         backend = 'cpu'
 elif args.use_directml:
     backend = 'directml'
     from modules.dml import directml_init
     ok, e = directml_init()
     if not ok:
-        log.error('DirectML initialization failed: {e}')
+        log.error(f'DirectML initialization failed: {e}')
         backend = 'cpu'
 elif torch.cuda.is_available() and torch.version.cuda:
     backend = 'cuda'

modules/extra_networks.py

@@ -1,6 +1,6 @@
 import re
 from collections import defaultdict
-from modules import errors
+from modules import errors, shared
 
 
 extra_network_registry = {}
@@ -62,17 +62,33 @@ class ExtraNetwork:
         raise NotImplementedError
 
 
-def activate(p, extra_network_data):
+def is_stepwise(en_obj):
+    all_args = []
+    for en in en_obj:
+        all_args.extend(en.positional[1:])
+        all_args.extend(en.named.values())
+    return any([len(str(x).split("@")) > 1 for x in all_args]) # noqa C419
+
+
+def activate(p, extra_network_data, step=0):
     """call activate for extra networks in extra_network_data in specified order, then call activate for all remaining registered networks with an empty argument list"""
     if extra_network_data is None:
         return
+    stepwise = False
+    for extra_network_args in extra_network_data.values():
+        stepwise = stepwise or is_stepwise(extra_network_args)
+    functional = shared.opts.lora_functional
+    if shared.opts.lora_force_diffusers and stepwise:
+        shared.log.warning("Composable LoRA not compatible with 'lora_force_diffusers'")
+        stepwise = False
+    shared.opts.data['lora_functional'] = stepwise or functional
     for extra_network_name, extra_network_args in extra_network_data.items():
         extra_network = extra_network_registry.get(extra_network_name, None)
         if extra_network is None:
             errors.log.warning(f"Skipping unknown extra network: {extra_network_name}")
             continue
         try:
-            extra_network.activate(p, extra_network_args)
+            extra_network.activate(p, extra_network_args, step=step)
         except Exception as e:
             errors.display(e, f"activating extra network: name={extra_network_name} args:{extra_network_args}")
 
@@ -84,6 +100,9 @@ def activate(p, extra_network_data):
             extra_network.activate(p, [])
         except Exception as e:
             errors.display(e, f"activating extra network: name={extra_network_name}")
+    if stepwise:
+        p.extra_network_data = extra_network_data
+        shared.opts.data['lora_functional'] = functional
 
 
 def deactivate(p, extra_network_data):

modules/generation_parameters_copypaste.py

@@ -187,24 +187,34 @@ def send_image_and_dimensions(x):
     return img, w, h
 
 
-def parse_generation_parameters(x: str):
+def parse_generation_parameters(param_str: str):
     res = {}
-    if x is None:
+    if param_str is None:
         return res
-    remaining = x.replace('\n', ' ').strip()
+    remaining = param_str.replace('\n', ' ').strip()
+    # remaining = param_str.strip()
     if len(remaining) == 0:
         return res
-    remaining = x[7:] if x.startswith('Prompt: ') else x
-    remaining = x[11:] if x.startswith('parameters: ') else x
-    if 'Steps: ' in remaining and 'Negative prompt: ' not in remaining:
-        remaining = remaining.replace('Steps: ', 'Negative prompt: Steps: ')
-    prompt, remaining = remaining.strip().split('Negative prompt: ', maxsplit=1) if 'Negative prompt: ' in remaining else (remaining, '')
+    if 'prompt:' in remaining:
+        remaining = remaining.replace('prompt:', 'Prompt:')
+    if 'negative prompt:' in remaining:
+        remaining = remaining.replace('negative prompt:', 'Negative prompt:')
+    if 'Negative Prompt:' in remaining:
+        remaining = remaining.replace('Negative Prompt:', 'Negative prompt:')
+    if 'steps:' in remaining:
+        remaining = remaining.replace('steps:', 'Steps:')
+    remaining = remaining[7:] if remaining.startswith('Prompt:') else remaining
+    remaining = remaining[11:] if remaining.startswith('parameters:') else remaining
+    if 'Steps:' in remaining and 'Negative prompt:' not in remaining:
+        remaining = remaining.replace('Steps:', 'Negative prompt: Steps:')
+    prompt, remaining = remaining.strip().split('Negative prompt:', maxsplit=1) if 'Negative prompt:' in remaining else (remaining, '')
     res["Prompt"] = prompt.strip()
-    negative, remaining = remaining.strip().split('Steps: ', maxsplit=1) if 'Steps: ' in remaining else (remaining, None)
+    negative, remaining = remaining.strip().split('Steps:', maxsplit=1) if 'Steps:' in remaining else (remaining, None)
     res["Negative prompt"] = negative.strip()
+
     if remaining is None:
         return res
-    remaining = f'Steps: {remaining}'
+    remaining = f'Steps: {remaining.strip()}'
     for k, v in re_param.findall(remaining.strip()):
         try:
             if v[0] == '"' and v[-1] == '"':

modules/images.py

@@ -7,6 +7,7 @@ import json
 import uuid
 import queue
 import string
+import random
 import hashlib
 import datetime
 import threading
@@ -253,6 +254,7 @@ def resize_image(resize_mode, im, width, height, upscaler_name=None, output_type
                 if upscaler is not None:
                     im = latent(im, w, h, upscaler)
                 else:
+                    upscaler = upscalers[0]
                     shared.log.warning(f"Resize upscaler: invalid={upscaler_name} fallback={upscaler.name}")
         if im.width != w or im.height != h: # probably downsample after upscaler created larger image
             im = im.resize((w, h), resample=Image.Resampling.LANCZOS)
@@ -534,9 +536,9 @@ def atomically_save_image():
         try:
             image_format = Image.registered_extensions()[extension]
         except Exception:
-            shared.log.warning(f'Unknown image format: {extension}')
+            shared.log.warning(f'Saving: unknown image format: {extension}')
             image_format = 'JPEG'
-        if shared.opts.image_watermark_enabled:
+        if shared.opts.image_watermark_enabled or (shared.opts.image_watermark_position != 'none' and shared.opts.image_watermark_image != ''):
             image = set_watermark(image, shared.opts.image_watermark)
         size = os.path.getsize(fn) if os.path.exists(fn) else 0
         shared.log.info(f'Saving: image="{fn}" type={image_format} resolution={image.width}x{image.height} size={size}')
@@ -547,42 +549,33 @@ def atomically_save_image():
                     file.write(f"{exifinfo}\n")
                 shared.log.info(f'Saving: text="{filename_txt}" len={len(exifinfo)}')
             except Exception as e:
-                shared.log.warning(f'Image description save failed: {filename_txt} {e}')
+                shared.log.warning(f'Saving failed: description={filename_txt} {e}')
         # actual save
         exifinfo = (exifinfo or "") if shared.opts.image_metadata else ""
         if image_format == 'PNG':
             pnginfo_data = PngImagePlugin.PngInfo()
             for k, v in params.pnginfo.items():
                 pnginfo_data.add_text(k, str(v))
-            try:
-                image.save(fn, format=image_format, compress_level=6, pnginfo=pnginfo_data if shared.opts.image_metadata else None)
-            except Exception as e:
-                shared.log.error(f'Image save failed: file="{fn}" {e}')
+            save_args = { 'compress_level': 6, 'pnginfo': pnginfo_data if shared.opts.image_metadata else None }
         elif image_format == 'JPEG':
             if image.mode == 'RGBA':
-                shared.log.warning('Saving RGBA image as JPEG: Alpha channel will be lost')
+                shared.log.warning('Saving: removing alpha channel')
                 image = image.convert("RGB")
             elif image.mode == 'I;16':
                 image = image.point(lambda p: p * 0.0038910505836576).convert("L")
             exif_bytes = piexif.dump({ "Exif": { piexif.ExifIFD.UserComment: piexif.helper.UserComment.dump(exifinfo, encoding="unicode") } })
-            try:
-                image.save(fn, format=image_format, optimize=True, quality=shared.opts.jpeg_quality, exif=exif_bytes)
-            except Exception as e:
-                shared.log.error(f'Image save failed: file="{fn}" {e}')
+            save_args = { 'optimize': True, 'quality': shared.opts.jpeg_quality, 'exif': exif_bytes if shared.opts.image_metadata else None }
         elif image_format == 'WEBP':
             if image.mode == 'I;16':
                 image = image.point(lambda p: p * 0.0038910505836576).convert("RGB")
             exif_bytes = piexif.dump({ "Exif": { piexif.ExifIFD.UserComment: piexif.helper.UserComment.dump(exifinfo, encoding="unicode") } })
-            try:
-                image.save(fn, format=image_format, quality=shared.opts.jpeg_quality, lossless=shared.opts.webp_lossless, exif=exif_bytes)
-            except Exception as e:
-                shared.log.error(f'Image save failed: file="{fn}" {e}')
+            save_args = { 'optimize': True, 'quality': shared.opts.jpeg_quality, 'exif': exif_bytes if shared.opts.image_metadata else None, 'lossless': shared.opts.webp_lossless }
         else:
-            # shared.log.warning(f'Unrecognized image format: {extension} attempting save as {image_format}')
-            try:
-                image.save(fn, format=image_format, quality=shared.opts.jpeg_quality)
-            except Exception as e:
-                shared.log.error(f'Image save failed: file="{fn}" {e}')
+            save_args = { 'quality': shared.opts.jpeg_quality }
+        try:
+            image.save(fn, format=image_format, **save_args)
+        except Exception as e:
+            shared.log.error(f'Saving failed: file="{fn}" format={image_format} {e}')
         if shared.opts.save_log_fn != '' and len(exifinfo) > 0:
             fn = os.path.join(paths.data_path, shared.opts.save_log_fn)
             if not fn.endswith('.json'):
@@ -839,24 +832,55 @@ def flatten(img, bgcolor):
 
 
 def set_watermark(image, watermark):
-    from imwatermark import WatermarkEncoder
-    wm_type = 'bytes'
-    wm_method = 'dwtDctSvd'
-    wm_length = 32
-    length = wm_length // 8
-    info = image.info
-    data = np.asarray(image)
-    encoder = WatermarkEncoder()
-    text = f"{watermark:<{length}}"[:length]
-    bytearr = text.encode(encoding='ascii', errors='ignore')
-    try:
-        encoder.set_watermark(wm_type, bytearr)
-        encoded = encoder.encode(data, wm_method)
-        image = Image.fromarray(encoded)
-        image.info = info
-        shared.log.debug(f'Set watermark: {watermark} method={wm_method} bits={wm_length}')
-    except Exception as e:
-        shared.log.warning(f'Set watermark error: {watermark} method={wm_method} bits={wm_length} {e}')
+    if shared.opts.image_watermark_position != 'none': # visible watermark
+        wm_image = None
+        try:
+            wm_image = Image.open(shared.opts.image_watermark_image)
+        except Exception as e:
+            shared.log.warning(f'Set image watermark: fn="{shared.opts.image_watermark_image}" {e}')
+        if wm_image is not None:
+            if shared.opts.image_watermark_position == 'top/left':
+                position = (0, 0)
+            elif shared.opts.image_watermark_position == 'top/right':
+                position = (image.width - wm_image.width, 0)
+            elif shared.opts.image_watermark_position == 'bottom/left':
+                position = (0, image.height - wm_image.height)
+            elif shared.opts.image_watermark_position == 'bottom/right':
+                position = (image.width - wm_image.width, image.height - wm_image.height)
+            elif shared.opts.image_watermark_position == 'center':
+                position = ((image.width - wm_image.width) // 2, (image.height - wm_image.height) // 2)
+            else:
+                position = (random.randint(0, image.width - wm_image.width), random.randint(0, image.height - wm_image.height))
+            try:
+                for x in range(wm_image.width):
+                    for y in range(wm_image.height):
+                        r, g, b, _a = wm_image.getpixel((x, y))
+                        if not (r == 0 and g == 0 and b == 0):
+                            image.putpixel((x+position[0], y+position[1]), (r, g, b))
+                shared.log.debug(f'Set image watermark: fn="{shared.opts.image_watermark_image}" image={wm_image} position={position}')
+            except Exception as e:
+                shared.log.warning(f'Set image watermark: image={wm_image} {e}')
+
+    if shared.opts.image_watermark_enabled: # invisible watermark
+        from imwatermark import WatermarkEncoder
+        wm_type = 'bytes'
+        wm_method = 'dwtDctSvd'
+        wm_length = 32
+        length = wm_length // 8
+        info = image.info
+        data = np.asarray(image)
+        encoder = WatermarkEncoder()
+        text = f"{watermark:<{length}}"[:length]
+        bytearr = text.encode(encoding='ascii', errors='ignore')
+        try:
+            encoder.set_watermark(wm_type, bytearr)
+            encoded = encoder.encode(data, wm_method)
+            image = Image.fromarray(encoded)
+            image.info = info
+            shared.log.debug(f'Set invisible watermark: {watermark} method={wm_method} bits={wm_length}')
+        except Exception as e:
+            shared.log.warning(f'Set invisible watermark error: {watermark} method={wm_method} bits={wm_length} {e}')
+
     return image
 
 

modules/interrogate.py

@@ -79,7 +79,7 @@ class InterrogateModels:
 
     def load_blip_model(self):
         self.create_fake_fairscale()
-        from repositories.blip import models
+        from repositories.blip import models # pylint: disable=unused-import
         from repositories.blip.models import blip
         import modules.modelloader as modelloader
         model_path = os.path.join(paths.models_path, "BLIP")
@@ -195,3 +195,162 @@ class InterrogateModels:
         self.unload()
         shared.state.end()
         return res
+
+# --------- interrrogate ui
+
+ci = None
+low_vram = False
+
+
+class BatchWriter:
+    def __init__(self, folder):
+        self.folder = folder
+        self.csv, self.file = None, None
+
+    def add(self, file, prompt):
+        txt_file = os.path.splitext(file)[0] + ".txt"
+        with open(os.path.join(self.folder, txt_file), 'w', encoding='utf-8') as f:
+            f.write(prompt)
+
+    def close(self):
+        if self.file is not None:
+            self.file.close()
+
+
+def get_clip_models():
+    import open_clip
+    return ['/'.join(x) for x in open_clip.list_pretrained()]
+
+
+def load_interrogator(model):
+    from clip_interrogator import Config, Interrogator
+    global ci # pylint: disable=global-statement
+    if ci is None:
+        config = Config(device=devices.get_optimal_device(), cache_path=os.path.join(paths.models_path, 'Interrogator'), clip_model_name=model, quiet=True)
+        if low_vram:
+            config.apply_low_vram_defaults()
+        shared.log.info(f'Interrogate load: config={config}')
+        ci = Interrogator(config)
+    elif model != ci.config.clip_model_name:
+        ci.config.clip_model_name = model
+        shared.log.info(f'Interrogate load: config={ci.config}')
+        ci.load_clip_model()
+
+
+def unload_clip_model():
+    if ci is not None:
+        shared.log.debug('Interrogate offload')
+        ci.caption_model = ci.caption_model.to(devices.cpu)
+        ci.clip_model = ci.clip_model.to(devices.cpu)
+        ci.caption_offloaded = True
+        ci.clip_offloaded = True
+        devices.torch_gc()
+
+
+def interrogate(image, mode, caption=None):
+    shared.log.info(f'Interrogate: image={image} mode={mode} config={ci.config}')
+    if mode == 'best':
+        prompt = ci.interrogate(image, caption=caption)
+    elif mode == 'caption':
+        prompt = ci.generate_caption(image) if caption is None else caption
+    elif mode == 'classic':
+        prompt = ci.interrogate_classic(image, caption=caption)
+    elif mode == 'fast':
+        prompt = ci.interrogate_fast(image, caption=caption)
+    elif mode == 'negative':
+        prompt = ci.interrogate_negative(image)
+    else:
+        raise RuntimeError(f"Unknown mode {mode}")
+    return prompt
+
+
+def interrogate_image(image, model, mode):
+    shared.state.begin()
+    shared.state.job = 'interrogate'
+    try:
+        if shared.backend == shared.Backend.ORIGINAL and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram):
+            lowvram.send_everything_to_cpu()
+            devices.torch_gc()
+        load_interrogator(model)
+        image = image.convert('RGB')
+        shared.log.info(f'Interrogate: image={image} mode={mode} config={ci.config}')
+        prompt = interrogate(image, mode)
+    except Exception as e:
+        prompt = f"Exception {type(e)}"
+        shared.log.error(f'Interrogate: {e}')
+    shared.state.end()
+    return prompt
+
+
+def interrogate_batch(batch_files, batch_folder, batch_str, model, mode, write):
+    files = []
+    if batch_files is not None:
+        files += [f.name for f in batch_files]
+    if batch_folder is not None:
+        files += [f.name for f in batch_folder]
+    if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str):
+        files += [os.path.join(batch_str, f) for f in os.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))]
+    if len(files) == 0:
+        shared.log.error('Interrogate batch no images')
+        return ''
+    shared.state.begin()
+    shared.state.job = 'batch interrogate'
+    prompts = []
+    try:
+        if shared.backend == shared.Backend.ORIGINAL and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram):
+            lowvram.send_everything_to_cpu()
+            devices.torch_gc()
+        load_interrogator(model)
+        shared.log.info(f'Interrogate batch: images={len(files)} mode={mode} config={ci.config}')
+        captions = []
+        # first pass: generate captions
+        for file in files:
+            caption = ""
+            try:
+                if shared.state.interrupted:
+                    break
+                image = Image.open(file).convert('RGB')
+                caption = ci.generate_caption(image)
+            except Exception as e:
+                shared.log.error(f'Interrogate caption: {e}')
+            finally:
+                captions.append(caption)
+        # second pass: interrogate
+        if write:
+            writer = BatchWriter(os.path.dirname(files[0]))
+        for idx, file in enumerate(files):
+            try:
+                if shared.state.interrupted:
+                    break
+                image = Image.open(file).convert('RGB')
+                prompt = interrogate(image, mode, caption=captions[idx])
+                prompts.append(prompt)
+                if write:
+                    writer.add(file, prompt)
+            except OSError as e:
+                shared.log.error(f'Interrogate batch: {e}')
+        if write:
+            writer.close()
+        ci.config.quiet = False
+        unload_clip_model()
+    except Exception as e:
+        shared.log.error(f'Interrogate batch: {e}')
+    shared.state.end()
+    return '\n\n'.join(prompts)
+
+
+def analyze_image(image, model):
+    load_interrogator(model)
+    image = image.convert('RGB')
+    image_features = ci.image_to_features(image)
+    top_mediums = ci.mediums.rank(image_features, 5)
+    top_artists = ci.artists.rank(image_features, 5)
+    top_movements = ci.movements.rank(image_features, 5)
+    top_trendings = ci.trendings.rank(image_features, 5)
+    top_flavors = ci.flavors.rank(image_features, 5)
+    medium_ranks = dict(zip(top_mediums, ci.similarities(image_features, top_mediums)))
+    artist_ranks = dict(zip(top_artists, ci.similarities(image_features, top_artists)))
+    movement_ranks = dict(zip(top_movements, ci.similarities(image_features, top_movements)))
+    trending_ranks = dict(zip(top_trendings, ci.similarities(image_features, top_trendings)))
+    flavor_ranks = dict(zip(top_flavors, ci.similarities(image_features, top_flavors)))
+    return medium_ranks, artist_ranks, movement_ranks, trending_ranks, flavor_ranks

modules/ipadapter.py

@@ -23,10 +23,10 @@ ADAPTERS = {
     'Plus': 'ip-adapter-plus_sd15.safetensors',
     'Plus Face': 'ip-adapter-plus-face_sd15.safetensors',
     'Full Face': 'ip-adapter-full-face_sd15.safetensors',
-    'Base SXDL': 'ip-adapter_sdxl.safetensors',
-    'Base ViT-H SXDL': 'ip-adapter_sdxl_vit-h.safetensors',
-    'Plus ViT-H SXDL': 'ip-adapter-plus_sdxl_vit-h.safetensors',
-    'Plus Face ViT-H SXDL': 'ip-adapter-plus-face_sdxl_vit-h.safetensors',
+    'Base SDXL': 'ip-adapter_sdxl.safetensors',
+    'Base ViT-H SDXL': 'ip-adapter_sdxl_vit-h.safetensors',
+    'Plus ViT-H SDXL': 'ip-adapter-plus_sdxl_vit-h.safetensors',
+    'Plus Face ViT-H SDXL': 'ip-adapter-plus-face_sdxl_vit-h.safetensors',
 }
 
 

modules/masking.py

@@ -141,6 +141,7 @@ MODELS = {
     # "isnet-anime",
 }
 COLORMAP = ['autumn', 'bone', 'jet', 'winter', 'rainbow', 'ocean', 'summer', 'spring', 'cool', 'hsv', 'pink', 'hot', 'parula', 'magma', 'inferno', 'plasma', 'viridis', 'cividis', 'twilight', 'shifted', 'turbo', 'deepgreen']
+TYPES = ['None', 'Opaque', 'Binary', 'Masked', 'Grayscale', 'Color', 'Composite']
 cache_dir = 'models/control/segment'
 generator: MaskGenerationPipeline = None
 busy = False

modules/modelloader.py

@@ -87,7 +87,7 @@ def download_civit_model_thread(model_name, model_url, model_path, model_type, p
         model_file = os.path.join(shared.opts.ckpt_dir, model_path, model_name)
         temp_file = os.path.join(shared.opts.ckpt_dir, model_path, temp_file)
 
-    res = f'CivitAI download: name="{model_name}" url="{model_url}" path="{model_path}" temp="{temp_file}"'
+    res = f'Model download: name="{model_name}" url="{model_url}" path="{model_path}" temp="{temp_file}"'
     if os.path.isfile(model_file):
         res += ' already exists'
         shared.log.warning(res)
@@ -144,7 +144,7 @@ def download_civit_model(model_url: str, model_name: str, model_path: str, model
     import threading
     thread = threading.Thread(target=download_civit_model_thread, args=(model_name, model_url, model_path, model_type, preview, token))
     thread.start()
-    return f'CivitAI download: name={model_name} url={model_url} path={model_path}'
+    return f'Model download: name={model_name} url={model_url} path={model_path}'
 
 
 def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config: Dict[str, str] = None, token = None, variant = None, revision = None, mirror = None, custom_pipeline = None):
@@ -215,60 +215,47 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config
     return pipeline_dir
 
 
-def load_diffusers_models(model_path: str, command_path: str = None, clear=True):
+def load_diffusers_models(clear=True):
     excluded_models = [
         'PhotoMaker', 'inswapper_128', 'IP-Adapter'
     ]
     t0 = time.time()
-    places = []
-    places.append(model_path)
-    if command_path is not None and command_path != model_path:
-        places.append(command_path)
+    place = shared.opts.diffusers_dir
+    if place is None or len(place) == 0 or not os.path.isdir(place):
+        place = os.path.join(models_path, 'Diffusers')
     if clear:
         diffuser_repos.clear()
     output = []
-    for place in places:
-        if not os.path.isdir(place):
-            continue
-        try:
-            """
-            import huggingface_hub as hf
-            res = hf.scan_cache_dir(cache_dir=place)
-            for r in list(res.repos):
-                cache_path = os.path.join(r.repo_path, "snapshots", list(r.revisions)[-1].commit_hash)
-                diffuser_repos.append({ 'name': r.repo_id, 'filename': r.repo_id, 'path': cache_path, 'size': r.size_on_disk, 'mtime': r.last_modified, 'hash': list(r.revisions)[-1].commit_hash, 'model_info': str(os.path.join(cache_path, "model_info.json")) })
-                if not os.path.isfile(os.path.join(cache_path, "hidden")):
-                    output.append(str(r.repo_id))
-            """
-            for folder in os.listdir(place):
-                try:
-                    if any([x in folder for x in excluded_models]): # noqa:C419
-                        continue
-                    if "--" not in folder:
-                        continue
-                    if folder.endswith("-prior"):
-                        continue
-                    _, name = folder.split("--", maxsplit=1)
-                    name = name.replace("--", "/")
-                    folder = os.path.join(place, folder)
-                    friendly = os.path.join(place, name)
-                    snapshots = os.listdir(os.path.join(folder, "snapshots"))
-                    if len(snapshots) == 0:
-                        shared.log.warning(f"Diffusers folder has no snapshots: location={place} folder={folder} name={name}")
-                        continue
-                    commit = os.path.join(folder, 'snapshots', snapshots[-1])
-                    mtime = os.path.getmtime(commit)
-                    info = os.path.join(commit, "model_info.json")
-                    diffuser_repos.append({ 'name': name, 'filename': name, 'friendly': friendly, 'folder': folder, 'path': commit, 'hash': commit, 'mtime': mtime, 'model_info': info })
-                    if os.path.exists(os.path.join(folder, 'hidden')):
-                        continue
-                    output.append(name)
-                except Exception:
-                    # shared.log.error(f"Error analyzing diffusers model: {folder} {e}")
-                    pass
-        except Exception as e:
-            shared.log.error(f"Error listing diffusers: {place} {e}")
-    shared.log.debug(f'Scanning diffusers cache: {places} items={len(output)} time={time.time()-t0:.2f}')
+    try:
+        for folder in os.listdir(place):
+            try:
+                if any([x in folder for x in excluded_models]): # noqa:C419
+                    continue
+                if "--" not in folder:
+                    continue
+                if folder.endswith("-prior"):
+                    continue
+                _, name = folder.split("--", maxsplit=1)
+                name = name.replace("--", "/")
+                folder = os.path.join(place, folder)
+                friendly = os.path.join(place, name)
+                snapshots = os.listdir(os.path.join(folder, "snapshots"))
+                if len(snapshots) == 0:
+                    shared.log.warning(f"Diffusers folder has no snapshots: location={place} folder={folder} name={name}")
+                    continue
+                commit = os.path.join(folder, 'snapshots', snapshots[-1])
+                mtime = os.path.getmtime(commit)
+                info = os.path.join(commit, "model_info.json")
+                diffuser_repos.append({ 'name': name, 'filename': name, 'friendly': friendly, 'folder': folder, 'path': commit, 'hash': commit, 'mtime': mtime, 'model_info': info })
+                if os.path.exists(os.path.join(folder, 'hidden')):
+                    continue
+                output.append(name)
+            except Exception:
+                # shared.log.error(f"Error analyzing diffusers model: {folder} {e}")
+                pass
+    except Exception as e:
+        shared.log.error(f"Error listing diffusers: {place} {e}")
+    shared.log.debug(f'Scanning diffusers cache: folder={place} items={len(output)} time={time.time()-t0:.2f}')
     return output
 
 

modules/processing.py

@@ -6,7 +6,7 @@ import numpy as np
 from PIL import Image
 from modules import shared, devices, errors, images, scripts, memstats, lowvram, script_callbacks, extra_networks, face_restoration, sd_hijack_freeu, sd_models, sd_vae, processing_helpers
 from modules.sd_hijack_hypertile import context_hypertile_vae, context_hypertile_unet
-from modules.processing_class import StableDiffusionProcessing, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img # pylint: disable=unused-import
+from modules.processing_class import StableDiffusionProcessing, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, StableDiffusionProcessingControl # pylint: disable=unused-import
 from modules.processing_info import create_infotext
 
 
@@ -163,6 +163,11 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
             sd_models.apply_token_merging(p.sd_model, p.get_token_merging_ratio())
             sd_hijack_freeu.apply_freeu(p, shared.backend == shared.Backend.ORIGINAL)
 
+        if p.width is not None:
+            p.width = 8 * int(p.width / 8)
+        if p.height is not None:
+            p.height = 8 * int(p.height / 8)
+
         script_callbacks.before_process_callback(p)
 
         if shared.cmd_opts.profile:

modules/processing_class.py

@@ -1,5 +1,4 @@
 import os
-import math
 import hashlib
 from typing import Any, Dict, List
 from dataclasses import dataclass, field
@@ -237,10 +236,12 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
         self.width = self.width or 512
         self.height = self.height or 512
 
-    def init_hr(self):
+    def init_hr(self, scale = None, upscaler = None):
+        scale = scale or self.hr_scale
+        upscaler = upscaler or self.hr_upscaler
         if self.hr_resize_x == 0 and self.hr_resize_y == 0:
-            self.hr_upscale_to_x = int(self.width * self.hr_scale)
-            self.hr_upscale_to_y = int(self.height * self.hr_scale)
+            self.hr_upscale_to_x = int(self.width * scale)
+            self.hr_upscale_to_y = int(self.height * scale)
         else:
             if self.hr_resize_y == 0:
                 self.hr_upscale_to_x = self.hr_resize_x
@@ -262,7 +263,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
                 self.truncate_x = (self.hr_upscale_to_x - target_w) // 8
                 self.truncate_y = (self.hr_upscale_to_y - target_h) // 8
         if shared.backend == shared.Backend.ORIGINAL: # diffusers are handled in processing_diffusers
-            if (self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height) or self.hr_upscaler is None or self.hr_upscaler == 'None': # special case: the user has chosen to do nothing
+            if (self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height) or upscaler is None or upscaler == 'None': # special case: the user has chosen to do nothing
                 self.is_hr_pass = False
                 return
             self.is_hr_pass = True
@@ -283,6 +284,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
         self.resize_mode: int = resize_mode
         self.resize_name: str = resize_name
         self.denoising_strength: float = denoising_strength
+        self.hr_denoising_strength: float = denoising_strength
         self.image_cfg_scale: float = image_cfg_scale
         self.init_latent = None
         self.image_mask = mask
@@ -311,9 +313,9 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
         self.script_args = []
 
     def init(self, all_prompts, all_seeds, all_subseeds):
-        if shared.backend == shared.Backend.DIFFUSERS and self.image_mask is not None and not self.is_control:
+        if shared.backend == shared.Backend.DIFFUSERS and getattr(self, 'image_mask', None) is not None:
             shared.sd_model = sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.INPAINTING)
-        elif shared.backend == shared.Backend.DIFFUSERS and self.image_mask is None and not self.is_control:
+        elif shared.backend == shared.Backend.DIFFUSERS and getattr(self, 'init_images', None) is not None:
             shared.sd_model = sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
 
         if self.sampler_name == "PLMS":
@@ -477,14 +479,15 @@ class StableDiffusionProcessingControl(StableDiffusionProcessingImg2Img):
     def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): # abstract
         pass
 
-    def init_hr(self):
-        if self.resize_name == 'None' or self.scale_by == 1.0:
+    def init_hr(self, scale = None, upscaler = None):
+        scale = scale or self.scale_by
+        upscaler = upscaler or self.resize_name
+        if upscaler == 'None' or scale == 1.0:
             return
         self.is_hr_pass = True
         self.hr_force = True
-        self.hr_upscaler = self.resize_name
-        self.hr_upscale_to_x, self.hr_upscale_to_y = int(self.width * self.scale_by), int(self.height * self.scale_by)
-        self.hr_upscale_to_x, self.hr_upscale_to_y = 8 * math.ceil(self.hr_upscale_to_x / 8), 8 * math.ceil(self.hr_upscale_to_y / 8)
+        self.hr_upscaler = upscaler
+        self.hr_upscale_to_x, self.hr_upscale_to_y = 8 * int(self.width * scale / 8), 8 * int(self.height * scale / 8)
         # hypertile_set(self, hr=True)
         shared.state.job_count = 2 * self.n_iter
-        shared.log.debug(f'Control hires: upscaler="{self.hr_upscaler}" upscale={self.scale_by} size={self.hr_upscale_to_x}x{self.hr_upscale_to_y}')
+        shared.log.debug(f'Control hires: upscaler="{self.hr_upscaler}" upscale={scale} size={self.hr_upscale_to_x}x{self.hr_upscale_to_y}')

modules/processing_diffusers.py

@@ -8,7 +8,7 @@ import numpy as np
 import torch
 import torchvision.transforms.functional as TF
 import diffusers
-from modules import shared, devices, processing, sd_samplers, sd_models, images, errors, prompt_parser_diffusers, sd_hijack_hypertile, processing_correction, processing_vae, sd_models_compile
+from modules import shared, devices, processing, sd_samplers, sd_models, images, errors, prompt_parser_diffusers, sd_hijack_hypertile, processing_correction, processing_vae, sd_models_compile, extra_networks
 from modules.processing_helpers import resize_init_images, resize_hires, fix_prompts, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps
 from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed
 
@@ -73,6 +73,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
                 if shared.state.interrupted or shared.state.skipped:
                     raise AssertionError('Interrupted...')
                 time.sleep(0.1)
+        if hasattr(p, "extra_network_data"):
+            extra_networks.activate(p, p.extra_network_data, step=step)
         if latents is None:
             return kwargs
         elif shared.opts.nan_skip:
@@ -243,13 +245,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
                 args["prior_guidance_scale"] = p.cfg_scale
             if 'decoder_guidance_scale' in possible:
                 args["decoder_guidance_scale"] = p.image_cfg_scale
-            # TODO Stable Cascade callbacks are currently broken in combined pipeline so preview will not get triggered
-            if 'prior_callback_on_step_end' in possible:
-                possible.remove('callback_on_step_end')
-            if 'callback_on_step_end' in possible:
-                possible.remove('callback_on_step_end')
-            if 'callback' in possible:
-                possible.remove('callback')
 
         # set callbacks
         if 'callback_steps' in possible:
@@ -349,7 +344,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
         if p.sag_scale > 0 and is_txt2img():
             update_sampler(shared.sd_model)
             supported = ['DDIMScheduler', 'PNDMScheduler', 'DDPMScheduler', 'DEISMultistepScheduler', 'UniPCMultistepScheduler', 'DPMSolverMultistepScheduler', 'DPMSolverSinlgestepScheduler']
-            if sd_model.scheduler.__class__.__name__ in supported:
+            if hasattr(sd_model, 'sfast'):
+                shared.log.warning(f'SAG incompatible compile mode: backend={shared.opts.cuda_compile_backend}')
+            elif sd_model.scheduler.__class__.__name__ in supported:
                 sd_model = sd_models.switch_pipe(diffusers.StableDiffusionSAGPipeline, sd_model)
                 p.extra_generation_params["SAG scale"] = p.sag_scale
                 p.task_args['sag_scale'] = p.sag_scale
@@ -363,6 +360,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
             orig_pipeline = sd_model # processed ONNX pipeline should not be replaced with original pipeline.
         return sd_model
 
+    # sanitize init_images
+    if hasattr(p, 'init_images') and getattr(p, 'init_images', None) is None:
+        del p.init_images
     if len(getattr(p, 'init_images', [])) > 0:
         while len(p.init_images) < len(p.prompts):
             p.init_images.append(p.init_images[-1])
@@ -413,6 +413,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
     try:
         t0 = time.time()
         sd_models_compile.check_deepcache(enable=True)
+        sd_models.move_model(shared.sd_model, devices.device)
         output = shared.sd_model(**base_args) # pylint: disable=not-callable
         if isinstance(output, dict):
             output = SimpleNamespace(**output)
@@ -441,8 +442,10 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
         shared.log.error(f'Processing: args={base_args} {e}')
         errors.display(e, 'Processing')
 
-    if hasattr(shared.sd_model, 'embedding_db') and len(shared.sd_model.embedding_db.embeddings_used) > 0:
+    if hasattr(shared.sd_model, 'embedding_db') and len(shared.sd_model.embedding_db.embeddings_used) > 0: # register used embeddings
         p.extra_generation_params['Embeddings'] = ', '.join(shared.sd_model.embedding_db.embeddings_used)
+    if hasattr(p, 'task_args') and p.task_args.get('image', None) is not None: # replace input with output so it can be used by hires/refine
+        p.task_args['image'] = output.images
 
     shared.state.nextjob()
     if shared.state.interrupted or shared.state.skipped:
@@ -450,12 +453,15 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
         return results
 
     # optional second pass
-    if p.enable_hr and len(getattr(p, 'init_images', [])) == 0:
+    if p.enable_hr:
         p.is_hr_pass = True
-    if p.is_hr_pass:
-        p.init_hr()
+        p.init_hr(p.hr_scale, p.hr_upscaler)
         prev_job = shared.state.job
 
+        # hires runs on original pipeline
+        if hasattr(shared.sd_model, 'restore_pipeline') and shared.sd_model.restore_pipeline is not None:
+            shared.sd_model.restore_pipeline()
+
         # upscale
         if hasattr(p, 'height') and hasattr(p, 'width') and p.hr_upscaler is not None and p.hr_upscaler != 'None':
             shared.log.info(f'Upscale: upscaler="{p.hr_upscaler}" resize={p.hr_resize_x}x{p.hr_resize_y} upscale={p.hr_upscale_to_x}x{p.hr_upscale_to_y}')
@@ -467,7 +473,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
             sd_hijack_hypertile.hypertile_set(p, hr=True)
 
         latent_upscale = shared.latent_upscale_modes.get(p.hr_upscaler, None)
-        if (latent_upscale is not None or p.hr_force) and p.denoising_strength > 0:
+        if (latent_upscale is not None or p.hr_force) and getattr(p, 'hr_denoising_strength', p.denoising_strength) > 0:
             p.ops.append('hires')
             sd_models_compile.openvino_recompile_model(p, hires=True, refiner=False)
             if shared.sd_model.__class__.__name__ == "OnnxRawPipeline":
@@ -480,6 +486,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
             shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
             update_sampler(shared.sd_model, second_pass=True)
             shared.log.info(f'HiRes: class={shared.sd_model.__class__.__name__} sampler="{p.hr_sampler_name}"')
+            sd_models.move_model(shared.sd_model, devices.device)
+            orig_denoise = p.denoising_strength
+            p.denoising_strength = getattr(p, 'hr_denoising_strength', p.denoising_strength)
             hires_args = set_pipeline_args(
                 model=shared.sd_model,
                 prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts,
@@ -507,7 +516,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
                 sd_models_compile.openvino_post_compile(op="base")
             except AssertionError as e:
                 shared.log.info(e)
-            p.init_images = []
+            p.denoising_strength = orig_denoise
+            # p.init_images = []
         shared.state.job = prev_job
         shared.state.nextjob()
         p.is_hr_pass = False
@@ -561,8 +571,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
             )
             shared.state.sampling_steps = refiner_args['num_inference_steps']
             try:
-                if 'requires_aesthetics_score' in shared.sd_refiner.config:
-                    shared.sd_refiner.register_to_config(requires_aesthetics_score=shared.opts.diffusers_aesthetics_score)
+                if 'requires_aesthetics_score' in shared.sd_refiner.config: # sdxl-model needs false and sdxl-refiner needs true
+                    shared.sd_refiner.register_to_config(requires_aesthetics_score = getattr(shared.sd_refiner, 'tokenizer', None) is None)
                 refiner_output = shared.sd_refiner(**refiner_args) # pylint: disable=not-callable
                 if isinstance(refiner_output, dict):
                     refiner_output = SimpleNamespace(**refiner_output)

modules/prompt_parser_diffusers.py

@@ -151,13 +151,11 @@ def prepare_embedding_providers(pipe, clip_skip):
         embedding_type = -(clip_skip + 1)
     else:
         embedding_type = clip_skip
-    if hasattr(pipe, "tokenizer") and hasattr(pipe, "text_encoder"):
-        provider = EmbeddingsProvider(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder, truncate=False,
-                                      returned_embeddings_type=embedding_type, device=device)
+    if getattr(pipe, "tokenizer", None) is not None and getattr(pipe, "text_encoder", None) is not None:
+        provider = EmbeddingsProvider(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder, truncate=False, returned_embeddings_type=embedding_type, device=device)
         embeddings_providers.append(provider)
-    if hasattr(pipe, "tokenizer_2") and hasattr(pipe, "text_encoder_2"):
-        provider = EmbeddingsProvider(tokenizer=pipe.tokenizer_2, text_encoder=pipe.text_encoder_2, truncate=False,
-                                      returned_embeddings_type=embedding_type, device=device)
+    if getattr(pipe, "tokenizer_2", None) is not None and getattr(pipe, "text_encoder_2", None) is not None:
+        provider = EmbeddingsProvider(tokenizer=pipe.tokenizer_2, text_encoder=pipe.text_encoder_2, truncate=False, returned_embeddings_type=embedding_type, device=device)
         embeddings_providers.append(provider)
     return embeddings_providers
 

modules/sd_models.py

@@ -23,7 +23,6 @@ from ldm.util import instantiate_from_config
 from modules import paths, shared, shared_items, shared_state, modelloader, devices, script_callbacks, sd_vae, errors, hashes, sd_models_config, sd_models_compile, sd_hijack_accelerate
 from modules.timer import Timer
 from modules.memstats import memory_stats
-from modules.paths import models_path, script_path
 from modules.modeldata import model_data
 
 
@@ -48,7 +47,7 @@ class CheckpointInfo:
         self.filename = filename
         self.type = ''
         relname = filename
-        app_path = os.path.abspath(script_path)
+        app_path = os.path.abspath(paths.script_path)
 
         def rel(fn, path):
             try:
@@ -64,8 +63,8 @@ class CheckpointInfo:
             relname = rel(filename, shared.opts.diffusers_dir)
         elif relname.startswith(model_path):
             relname = rel(filename, model_path)
-        elif relname.startswith(script_path):
-            relname = rel(filename, script_path)
+        elif relname.startswith(paths.script_path):
+            relname = rel(filename, paths.script_path)
         elif relname.startswith(app_path):
             relname = rel(filename, app_path)
         else:
@@ -147,7 +146,7 @@ def list_models():
         ext_filter = [".ckpt", ".safetensors"]
     model_list = list(modelloader.load_models(model_path=model_path, model_url=None, command_path=shared.opts.ckpt_dir, ext_filter=ext_filter, download_name=None, ext_blacklist=[".vae.ckpt", ".vae.safetensors"]))
     if shared.backend == shared.Backend.DIFFUSERS:
-        model_list += modelloader.load_diffusers_models(model_path=os.path.join(models_path, 'Diffusers'), command_path=shared.opts.diffusers_dir, clear=True)
+        model_list += modelloader.load_diffusers_models(clear=True)
     for filename in sorted(model_list, key=str.lower):
         checkpoint_info = CheckpointInfo(filename)
         if checkpoint_info.name is not None:
@@ -683,7 +682,10 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model'):
                 shared.opts.diffusers_move_unet = False
                 shared.opts.diffusers_move_refiner = False
                 shared.log.warning(f'Disabling {op} "Move model to CPU" since "Model CPU offload" is enabled')
-            sd_model.enable_model_cpu_offload()
+            if not hasattr(sd_model, "_all_hooks") or len(sd_model._all_hooks) == 0: # pylint: disable=protected-access
+                sd_model.enable_model_cpu_offload()
+            else:
+                sd_model.maybe_free_model_hooks()
             sd_model.has_accelerate = True
     if hasattr(sd_model, "enable_sequential_cpu_offload"):
         if shared.cmd_opts.lowvram or shared.opts.diffusers_seq_cpu_offload:
@@ -746,21 +748,35 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model'):
 
 
 def move_model(model, device=None, force=False):
-    if model is not None:
-        if getattr(model, 'vae', None) is not None and get_diffusers_task(model) != DiffusersTaskType.TEXT_2_IMAGE:
-            if device == devices.device: # force vae back to gpu if not in txt2img mode
-                model.vae.to(device)
-                if hasattr(model.vae, '_hf_hook'):
-                    debug_move(f'Model move: to={device} class={model.vae.__class__} function={sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access
-                    model.vae._hf_hook.execution_device = device # pylint: disable=protected-access
-        if getattr(model, 'has_accelerate', False) and not force:
-            return
-        debug_move(f'Model move: to={device} class={model.__class__} function={sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access
-        try:
-            model.to(device)
-        except Exception as e:
-            shared.log.error(f'Model move: {e}')
-        devices.torch_gc()
+    if model is None or device is None:
+        return
+    if getattr(model, 'vae', None) is not None and get_diffusers_task(model) != DiffusersTaskType.TEXT_2_IMAGE:
+        if device == devices.device: # force vae back to gpu if not in txt2img mode
+            model.vae.to(device)
+            if hasattr(model.vae, '_hf_hook'):
+                debug_move(f'Model move: to={device} class={model.vae.__class__} function={sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access
+                model.vae._hf_hook.execution_device = device # pylint: disable=protected-access
+    debug_move(f'Model move: device={device} class={model.__class__} accelerate={getattr(model, "has_accelerate", False)} function={sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access
+    if hasattr(model, "components"): # accelerate patch
+        for name, m in model.components.items():
+            if not hasattr(m, "_hf_hook"): # not accelerate hook
+                break
+            if not isinstance(m, torch.nn.Module) or name in model._exclude_from_cpu_offload: # pylint: disable=protected-access
+                continue
+            for module in m.modules():
+                if (hasattr(module, "_hf_hook") and hasattr(module._hf_hook, "execution_device") and module._hf_hook.execution_device is not None): # pylint: disable=protected-access
+                    try:
+                        module._hf_hook.execution_device = device # pylint: disable=protected-access
+                    except Exception as e:
+                        if os.environ.get('SD_MOVE_DEBUG', None):
+                            shared.log.error(f'Model move execution device: device={device} {e}')
+    if getattr(model, 'has_accelerate', False) and not force:
+        return
+    try:
+        model.to(device)
+    except Exception as e:
+        shared.log.error(f'Model move: device={device} {e}')
+    devices.torch_gc()
 
 
 def get_load_config(model_file, model_type):
@@ -903,9 +919,9 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
                     shared.log.debug(f'StableCascade experimental: args={diffusers_load_config} device={devices.device} dtype={devices.dtype}')
                     diffusers_load_config.pop("vae", None)
                     diffusers_load_config.pop("variant", None)
-                    decoder = diffusers.StableCascadeDecoderPipeline.from_pretrained("stabilityai/stable-cascade", cache_dir=shared.opts.diffusers_dir, revision="refs/pr/17", **diffusers_load_config)
+                    decoder = diffusers.StableCascadeDecoderPipeline.from_pretrained("stabilityai/stable-cascade", cache_dir=shared.opts.diffusers_dir, revision="refs/pr/44", **diffusers_load_config)
                     shared.log.debug(f'StableCascade decoder: scale={decoder.latent_dim_scale}')
-                    prior = diffusers.StableCascadePriorPipeline.from_pretrained("stabilityai/stable-cascade-prior", cache_dir=shared.opts.diffusers_dir, **diffusers_load_config)
+                    prior = diffusers.StableCascadePriorPipeline.from_pretrained("stabilityai/stable-cascade-prior", cache_dir=shared.opts.diffusers_dir, revision="refs/pr/2", **diffusers_load_config)
                     shared.log.debug(f'StableCascade prior: scale={prior.resolution_multiple}')
                     sd_model = diffusers.StableCascadeCombinedPipeline(
                         tokenizer=decoder.tokenizer,
@@ -917,8 +933,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
                         prior_text_encoder=prior.text_encoder,
                         prior_tokenizer=prior.tokenizer,
                         prior_scheduler=prior.scheduler,
-                        prior_prior_feature_extractor=prior.feature_extractor,
-                        prior_prior_image_encoder=prior.image_encoder)
+                        prior_feature_extractor=prior.feature_extractor,
+                        prior_image_encoder=prior.image_encoder)
                     shared.log.debug(f'StableCascade combined: {sd_model.__class__.__name__}')
                 except Exception as e:
                     shared.log.error(f'Diffusers Failed loading {op}: {checkpoint_info.path} {e}')
@@ -996,9 +1012,6 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
                 if model_type.startswith('Stable Diffusion'):
                     if shared.opts.diffusers_force_zeros:
                         diffusers_load_config['force_zeros_for_empty_prompt '] = shared.opts.diffusers_force_zeros
-                    if shared.opts.diffusers_aesthetics_score:
-                        diffusers_load_config['requires_aesthetics_score'] = shared.opts.diffusers_aesthetics_score
-                    # diffusers_load_config['config_files'] = get_load_config(checkpoint_info.path.lower())
                     diffusers_load_config['original_config_file'] = get_load_config(checkpoint_info.path, model_type)
                 if hasattr(pipeline, 'from_single_file'):
                     diffusers_load_config['use_safetensors'] = True
@@ -1039,8 +1052,6 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
         elif "Kandinsky" in sd_model.__class__.__name__:
             sd_model.scheduler.name = 'DDIM'
 
-        set_diffuser_options(sd_model, vae, op)
-
         base_sent_to_cpu=False
         if (shared.opts.cuda_compile and shared.opts.cuda_compile_backend != 'none') or shared.opts.ipex_optimize or shared.opts.nncf_compress_weights:
             if op == 'refiner' and not getattr(sd_model, 'has_accelerate', False):
@@ -1088,6 +1099,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
         if op == 'refiner' and base_sent_to_cpu:
             shared.log.debug('Moving base model back to GPU')
             move_model(model_data.sd_model, devices.device)
+        set_diffuser_options(sd_model, vae, op) # offloading should enabled after all
     except Exception as e:
         shared.log.error("Failed to load diffusers model")
         errors.display(e, "loading Diffusers model")
@@ -1226,6 +1238,9 @@ def switch_pipe(cls: diffusers.DiffusionPipeline, pipeline: diffusers.DiffusionP
 def set_diffuser_pipe(pipe, new_pipe_type):
     if get_diffusers_task(pipe) == new_pipe_type:
         return pipe
+    # skip specific pipelines
+    if pipe.__class__.__name__ == 'StableDiffusionReferencePipeline' or pipe.__class__.__name__ == 'StableDiffusionAdapterPipeline' or 'Onnx' in pipe.__class__.__name__:
+        return pipe
 
     sd_checkpoint_info = getattr(pipe, "sd_checkpoint_info", None)
     sd_model_checkpoint = getattr(pipe, "sd_model_checkpoint", None)
@@ -1235,10 +1250,6 @@ def set_diffuser_pipe(pipe, new_pipe_type):
     image_encoder = getattr(pipe, "image_encoder", None)
     feature_extractor = getattr(pipe, "feature_extractor", None)
 
-    # skip specific pipelines
-    if pipe.__class__.__name__ == 'StableDiffusionReferencePipeline' or pipe.__class__.__name__ == 'StableDiffusionAdapterPipeline' or 'Onnx' in pipe.__class__.__name__:
-        return pipe
-
     try:
         if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE:
             new_pipe = diffusers.AutoPipelineForText2Image.from_pipe(pipe)
@@ -1250,8 +1261,8 @@ def set_diffuser_pipe(pipe, new_pipe_type):
         shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}')
         return pipe
 
-    if pipe.__class__ == new_pipe.__class__:
-        return pipe
+    # if pipe.__class__ == new_pipe.__class__:
+    #    return pipe
     new_pipe.sd_checkpoint_info = sd_checkpoint_info
     new_pipe.sd_model_checkpoint = sd_model_checkpoint
     new_pipe.sd_model_hash = sd_model_hash
@@ -1262,7 +1273,7 @@ def set_diffuser_pipe(pipe, new_pipe_type):
     new_pipe.is_sdxl = getattr(pipe, 'is_sdxl', False) # a1111 compatibility item
     new_pipe.is_sd2 = getattr(pipe, 'is_sd2', False)
     new_pipe.is_sd1 = getattr(pipe, 'is_sd1', True)
-    shared.log.debug(f"Pipeline class change: original={pipe.__class__.__name__} target={new_pipe.__class__.__name__}")
+    shared.log.debug(f"Pipeline class change: original={pipe.__class__.__name__} target={new_pipe.__class__.__name__} device={pipe.device} fn={sys._getframe().f_back.f_code.co_name}") # pylint: disable=protected-access
     pipe = new_pipe
     return pipe
 

modules/sd_models_compile.py

@@ -164,11 +164,12 @@ def compile_stablefast(sd_model):
     try:
         t0 = time.time()
         sd_model = sf.compile(sd_model, config)
+        sd_model.sfast = True
         setup_logging() # compile messes with logging so reset is needed
         if shared.opts.cuda_compile_precompile:
             sd_model("dummy prompt")
         t1 = time.time()
-        shared.log.info(f"Model compile: task=Stable-fast config={config.__dict__} time={t1-t0:.2f}")
+        shared.log.info(f"Model compile: task='Stable-fast' config={config.__dict__} time={t1-t0:.2f}")
     except Exception as e:
         shared.log.info(f"Model compile: task=Stable-fast error: {e}")
     return sd_model
@@ -261,7 +262,7 @@ def compile_deepcache(sd_model):
     deepcache_worker = DeepCacheSDHelper(pipe=sd_model)
     deepcache_worker.set_params(cache_interval=shared.opts.deep_cache_interval, cache_branch_id=0)
     t1 = time.time()
-    shared.log.info(f"Model compile: task=DeepCache config={deepcache_worker.params} time={t1-t0:.2f}")
+    shared.log.info(f"Model compile: task='DeepCache' config={deepcache_worker.params} time={t1-t0:.2f}")
     # config={'cache_interval': 3, 'cache_layer_id': 0, 'cache_block_id': 0, 'skip_mode': 'uniform'} time=0.00
     return sd_model
 

modules/shared.py

@@ -68,6 +68,7 @@ restricted_opts = {
     "outdir_txt2img_samples",
     "outdir_img2img_samples",
     "outdir_extras_samples",
+    "outdir_control_samples",
     "outdir_grids",
     "outdir_txt2img_grids",
     "outdir_save",
@@ -325,6 +326,12 @@ elif devices.backend == "directml":
 else: # cuda, rocm, ipex
     cross_attention_optimization_default ="Scaled-Dot-Product"
 
+if devices.backend == "rocm":
+    sdp_options_default =  ['Memory attention', 'Math attention']
+#elif devices.backend == "zluda":
+#    sdp_options_default =  ['Math attention']
+else:
+    sdp_options_default = ['Flash attention', 'Memory attention', 'Math attention']
 
 options_templates.update(options_section(('sd', "Execution & Models"), {
     "sd_backend": OptionInfo(default_backend, "Execution backend", gr.Radio, {"choices": ["original", "diffusers"] }),
@@ -358,7 +365,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), {
 
     "cross_attention_sep": OptionInfo("<h2>Attention</h2>", "", gr.HTML),
     "cross_attention_optimization": OptionInfo(cross_attention_optimization_default, "Attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention(diffusers=backend == Backend.DIFFUSERS) }),
-    "sdp_options": OptionInfo(['Flash attention', 'Memory attention', 'Math attention'], "SDP options", gr.CheckboxGroup, {"choices": ['Flash attention', 'Memory attention', 'Math attention'] }),
+    "sdp_options": OptionInfo(sdp_options_default, "SDP options", gr.CheckboxGroup, {"choices": ['Flash attention', 'Memory attention', 'Math attention'] }),
     "xformers_options": OptionInfo(['Flash attention'], "xFormers options", gr.CheckboxGroup, {"choices": ['Flash attention'] }),
     "dynamic_attention_slice_rate": OptionInfo(4, "Dynamic Attention slicing rate in GB", gr.Slider, {"minimum": 0.1, "maximum": 16, "step": 0.1, "visible": backend == Backend.DIFFUSERS}),
     "sub_quad_sep": OptionInfo("<h3>Sub-quadratic options</h3>", "", gr.HTML, {"visible": backend == Backend.ORIGINAL}),
@@ -457,7 +464,6 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), {
     "diffusers_to_gpu": OptionInfo(False, "Load model directly to GPU"),
     "disable_accelerate": OptionInfo(False, "Disable accelerate"),
     "diffusers_force_zeros": OptionInfo(False, "Force zeros for prompts when empty", gr.Checkbox, {"visible": False}),
-    "diffusers_aesthetics_score": OptionInfo(False, "Require aesthetics score"),
     "diffusers_pooled": OptionInfo("default", "Diffusers SDXL pooled embeds", gr.Radio, {"choices": ['default', 'weighted']}),
     "huggingface_token": OptionInfo('', 'HuggingFace token'),
 
@@ -514,8 +520,6 @@ options_templates.update(options_section(('saving-images', "Image Options"), {
     "image_metadata": OptionInfo(True, "Include metadata"),
     "save_txt": OptionInfo(False, "Create info file per image"),
     "save_log_fn": OptionInfo("", "Update JSON log file per image", component_args=hide_dirs),
-    "image_watermark_enabled": OptionInfo(False, "Include watermark"),
-    "image_watermark": OptionInfo('', "Watermark string"),
     "image_sep_grid": OptionInfo("<h2>Grid Options</h2>", "", gr.HTML),
     "grid_save": OptionInfo(True, "Save all generated image grids"),
     "grid_format": OptionInfo('jpg', 'File format', gr.Dropdown, {"choices": ["jpg", "png", "webp", "tiff", "jp2"]}),
@@ -532,6 +536,12 @@ options_templates.update(options_section(('saving-images', "Image Options"), {
     "save_images_before_color_correction": OptionInfo(False, "Save image before color correction"),
     "save_mask": OptionInfo(False, "Save inpainting mask"),
     "save_mask_composite": OptionInfo(False, "Save inpainting masked composite"),
+
+    "image_sep_watermark": OptionInfo("<h2>Watermarking</h2>", "", gr.HTML),
+    "image_watermark_enabled": OptionInfo(False, "Include invisible watermark"),
+    "image_watermark": OptionInfo('', "Invisible watermark string"),
+    "image_watermark_position": OptionInfo('none', 'Image watermark position', gr.Dropdown, {"choices": ["none", "top/left", "top/right", "bottom/left", "bottom/right", "center", "random"]}),
+    "image_watermark_image": OptionInfo('', "Image watermark file"),
 }))
 
 options_templates.update(options_section(('saving-paths', "Image Naming & Paths"), {
@@ -570,6 +580,7 @@ options_templates.update(options_section(('ui', "User Interface Options"), {
     "theme_style": OptionInfo("Auto", "Theme mode", gr.Radio, {"choices": ["Auto", "Dark", "Light"]}),
     "font_size": OptionInfo(14, "Font size", gr.Slider, {"minimum": 8, "maximum": 32, "step": 1, "visible": True}),
     "tooltips": OptionInfo("UI Tooltips", "UI tooltips", gr.Radio, {"choices": ["None", "Browser default", "UI tooltips"], "visible": False}),
+    "aspect_ratios": OptionInfo("1:1, 4:3, 16:9, 16:10, 21:9, 3:4, 9:16, 10:16, 9:21", "Allowed aspect ratios"),
     "compact_view": OptionInfo(False, "Compact view"),
     "return_grid": OptionInfo(True, "Show grid in results"),
     "return_mask": OptionInfo(False, "Inpainting include greyscale mask in results"),

modules/styles.py

@@ -5,17 +5,19 @@ import os
 import csv
 import json
 import time
+import random
 from installer import log
 from modules import files_cache
 
 
 class Style():
-    def __init__(self, name: str, desc: str = "", prompt: str = "", negative_prompt: str = "", extra: str = "", filename: str = "", preview: str = "", mtime: float = 0):
+    def __init__(self, name: str, desc: str = "", prompt: str = "", negative_prompt: str = "", extra: str = "", wildcards: str = "", filename: str = "", preview: str = "", mtime: float = 0):
         self.name = name
         self.description = desc
         self.prompt = prompt
         self.negative_prompt = negative_prompt
         self.extra = extra
+        self.wildcards = wildcards
         self.filename = filename
         self.preview = preview
         self.mtime = mtime
@@ -40,6 +42,21 @@ def apply_styles_to_prompt(prompt, styles):
     return prompt
 
 
+def apply_wildcards_to_prompt(prompt, all_wildcards):
+    replaced = {}
+    for style_wildcards in all_wildcards:
+        wildcards = [x.strip() for x in style_wildcards.split(";") if len(x.strip()) > 0]
+        for wildcard in wildcards:
+            what, words = wildcard.split("=", 1)
+            words = [x.strip() for x in words.split(",") if len(x.strip()) > 0]
+            word = random.choice(words)
+            prompt = prompt.replace(what, word)
+            replaced[what] = word
+    if replaced:
+        log.debug(f'Applying style wildcards: {replaced}')
+    return prompt
+
+
 def apply_styles_to_extra(p, style: Style):
     if style is None:
         return
@@ -47,10 +64,14 @@ def apply_styles_to_extra(p, style: Style):
         'sampler': 'sampler_name',
     }
     from modules.generation_parameters_copypaste import parse_generation_parameters
-    extra = parse_generation_parameters(style.extra)
+    s = style.extra
+    s = 'Negative prompt: ' + s if 'Negative prompt:' not in s else s
+    s = 'Prompt: ' + s if 'Prompt:' not in s else s
+    extra = parse_generation_parameters(s)
     extra.pop('Prompt', None)
     extra.pop('Negative prompt', None)
     fields = []
+    skipped = []
     for k, v in extra.items():
         k = k.lower()
         k = k.replace(' ', '_')
@@ -62,7 +83,9 @@ def apply_styles_to_extra(p, style: Style):
                 v = type(orig)(v)
             setattr(p, k, v)
             fields.append(f'{k}={v}')
-    log.debug(f'Applying style: name="{style.name}" extra={fields}')
+        else:
+            skipped.append(f'{k}={v}')
+    log.debug(f'Applying style: name="{style.name}" extra={fields} skipped={skipped}')
 
 
 class StyleDatabase:
@@ -115,6 +138,7 @@ class StyleDatabase:
                         prompt=style.get("prompt", ""),
                         negative_prompt=style.get("negative", ""),
                         extra=style.get("extra", ""),
+                        wildcards=style.get("wildcards", ""),
                         preview=style.get("preview", None),
                         filename=fn,
                         mtime=os.path.getmtime(fn),
@@ -171,13 +195,17 @@ class StyleDatabase:
         if styles is None or not isinstance(styles, list):
             log.error(f'Invalid styles: {styles}')
             return prompt
-        return apply_styles_to_prompt(prompt, [self.find_style(x).prompt for x in styles])
+        prompt = apply_styles_to_prompt(prompt, [self.find_style(x).prompt for x in styles])
+        prompt = apply_wildcards_to_prompt(prompt, [self.find_style(x).wildcards for x in styles])
+        return prompt
 
     def apply_negative_styles_to_prompt(self, prompt, styles):
         if styles is None or not isinstance(styles, list):
             log.error(f'Invalid styles: {styles}')
             return prompt
-        return apply_styles_to_prompt(prompt, [self.find_style(x).negative_prompt for x in styles])
+        prompt = apply_styles_to_prompt(prompt, [self.find_style(x).negative_prompt for x in styles])
+        prompt = apply_wildcards_to_prompt(prompt, [self.find_style(x).wildcards for x in styles])
+        return prompt
 
     def apply_styles_to_extra(self, p):
         if p.styles is None or not isinstance(p.styles, list):
@@ -221,7 +249,7 @@ class StyleDatabase:
                     name = row["name"]
                     prompt = row["prompt"] if "prompt" in row else row["text"]
                     negative = row.get("negative_prompt", "") if "negative_prompt" in row else row.get("negative", "")
-                    self.styles[name] = Style(name, desc=name, prompt=prompt, negative_prompt=negative, extra="")
+                    self.styles[name] = Style(name, desc=name, prompt=prompt, negative_prompt=negative)
                     log.debug(f'Migrated style: {self.styles[name].__dict__}')
                     num += 1
                 except Exception:

modules/theme.py

@@ -43,11 +43,12 @@ def list_themes():
     if 'sdnext-ui-ux' in extensions:
         ext = next((e for e in modules.extensions.extensions if e.name == 'sdnext-ui-ux'), None)
         folder = os.path.join(ext.path, 'themes')
-        engines.append('modern/default')
         if os.path.exists(folder):
             for f in os.listdir(folder):
                 if f.endswith('.css'):
                     engines.append(f'modern/{os.path.splitext(f)[0]}')
+        if len(engines) == 0:
+            engines.append('modern/sdxl_alpha')
     if 'sd-webui-lobe-theme' in extensions:
         modules.shared.log.info('Theme: installed="lobe"')
         engines.append('lobe')
@@ -68,6 +69,8 @@ def reload_gradio_theme(theme_name=None):
     theme_name = theme_name or modules.shared.cmd_opts.theme or modules.shared.opts.gradio_theme
     if theme_name == 'default':
         theme_name = 'black-teal'
+    if theme_name == 'modern':
+        theme_name = 'modern/sdxl_alpha'
     modules.shared.opts.data['gradio_theme'] = theme_name
     default_font_params = {
         'font':['Helvetica', 'ui-sans-serif', 'system-ui', 'sans-serif'],

modules/ui.py

@@ -3,7 +3,7 @@ import mimetypes
 import gradio as gr
 import gradio.routes
 import gradio.utils
-from modules.call_queue import wrap_gradio_call
+from modules.call_queue import wrap_gradio_call, wrap_gradio_gpu_call # pylint: disable=unused-import
 from modules import timer, gr_hijack, shared, theme, sd_models, script_callbacks, modelloader, ui_common, ui_loadsave, ui_symbols, ui_javascript, generation_parameters_copypaste, call_queue
 from modules.paths import script_path, data_path # pylint: disable=unused-import
 from modules.dml import directml_override_opts
@@ -149,12 +149,6 @@ def create_ui(startup_timer = None):
         ui_models.create_ui()
         timer.startup.record("ui-models")
 
-    with gr.Blocks(analytics_enabled=False) as interrogate_interface:
-        from modules import ui_interrogate
-        ui_interrogate.create_ui()
-        timer.startup.record("ui-interrogate")
-
-
     def create_setting_component(key, is_quicksettings=False):
         def fun():
             return opts.data[key] if key in opts.data else opts.data_labels[key].default
@@ -371,7 +365,6 @@ def create_ui(startup_timer = None):
     interfaces += [(img2img_interface, "Image", "img2img")]
     interfaces += [(control_interface, "Control", "control")] if control_interface is not None else []
     interfaces += [(extras_interface, "Process", "process")]
-    interfaces += [(interrogate_interface, "Interrogate", "interrogate")]
     interfaces += [(models_interface, "Models", "models")]
     interfaces += script_callbacks.ui_tabs_callback()
     interfaces += [(settings_interface, "System", "system")]

modules/ui_control.py

@@ -124,6 +124,8 @@ def create_ui(_blocks: gr.Blocks=None):
                         video_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False)
                     video_type.change(fn=helpers.video_type_change, inputs=[video_type], outputs=[video_duration, video_loop, video_pad, video_interpolate])
 
+                enable_hr, hr_sampler_index, hr_denoising_strength, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img')
+
             with gr.Row():
                 override_settings = ui_common.create_override_inputs('control')
 
@@ -403,54 +405,54 @@ def create_ui(_blocks: gr.Blocks=None):
                             if i == 0:
                                 units[-1].enabled = True # enable first unit in group
 
-                    with gr.Tab('Processor settings') as _tab_settings:
-                        with gr.Group(elem_classes=['processor-group']):
-                            settings = []
-                            with gr.Accordion('HED', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Checkbox(label="Scribble", value=False))
-                            with gr.Accordion('Midas depth', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Slider(label="Background threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1))
-                                settings.append(gr.Checkbox(label="Depth and normal", value=False))
-                            with gr.Accordion('MLSD', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Slider(label="Score threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1))
-                                settings.append(gr.Slider(label="Distance threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1))
-                            with gr.Accordion('OpenBody', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Checkbox(label="Body", value=True))
-                                settings.append(gr.Checkbox(label="Hands", value=False))
-                                settings.append(gr.Checkbox(label="Face", value=False))
-                            with gr.Accordion('PidiNet', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Checkbox(label="Scribble", value=False))
-                                settings.append(gr.Checkbox(label="Apply filter", value=False))
-                            with gr.Accordion('LineArt', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Checkbox(label="Coarse", value=False))
-                            with gr.Accordion('Leres Depth', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Checkbox(label="Boost", value=False))
-                                settings.append(gr.Slider(label="Near threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.0))
-                                settings.append(gr.Slider(label="Background threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.0))
-                            with gr.Accordion('MediaPipe Face', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Slider(label="Max faces", minimum=1, maximum=10, step=1, value=1))
-                                settings.append(gr.Slider(label="Min confidence", minimum=0.0, maximum=1.0, step=0.01, value=0.5))
-                            with gr.Accordion('Canny', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Slider(label="Low threshold", minimum=0, maximum=1000, step=1, value=100))
-                                settings.append(gr.Slider(label="High threshold", minimum=0, maximum=1000, step=1, value=200))
-                            with gr.Accordion('DWPose', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Radio(label="Model", choices=['Tiny', 'Medium', 'Large'], value='Tiny'))
-                                settings.append(gr.Slider(label="Min confidence", minimum=0.0, maximum=1.0, step=0.01, value=0.3))
-                            with gr.Accordion('SegmentAnything', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Radio(label="Model", choices=['Base', 'Large'], value='Base'))
-                            with gr.Accordion('Edge', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Checkbox(label="Parameter free", value=True))
-                                settings.append(gr.Radio(label="Mode", choices=['edge', 'gradient'], value='edge'))
-                            with gr.Accordion('Zoe Depth', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Checkbox(label="Gamma corrected", value=False))
-                            with gr.Accordion('Marigold Depth', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Dropdown(label="Color map", choices=['None'] + plt.colormaps(), value='None'))
-                                settings.append(gr.Slider(label="Denoising steps", minimum=1, maximum=99, step=1, value=10))
-                                settings.append(gr.Slider(label="Ensemble size", minimum=1, maximum=99, step=1, value=10))
-                            with gr.Accordion('Depth Anything', open=True, elem_classes=['processor-settings']):
-                                settings.append(gr.Dropdown(label="Color map", choices=['none'] + masking.COLORMAP, value='inferno'))
-                            for setting in settings:
-                                setting.change(fn=processors.update_settings, inputs=settings, outputs=[])
+                with gr.Accordion('Processor settings', open=False, elem_classes=['control-settings']) as _tab_settings:
+                    with gr.Group(elem_classes=['processor-group']):
+                        settings = []
+                        with gr.Accordion('HED', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Checkbox(label="Scribble", value=False))
+                        with gr.Accordion('Midas depth', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Slider(label="Background threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1))
+                            settings.append(gr.Checkbox(label="Depth and normal", value=False))
+                        with gr.Accordion('MLSD', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Slider(label="Score threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1))
+                            settings.append(gr.Slider(label="Distance threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1))
+                        with gr.Accordion('OpenBody', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Checkbox(label="Body", value=True))
+                            settings.append(gr.Checkbox(label="Hands", value=False))
+                            settings.append(gr.Checkbox(label="Face", value=False))
+                        with gr.Accordion('PidiNet', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Checkbox(label="Scribble", value=False))
+                            settings.append(gr.Checkbox(label="Apply filter", value=False))
+                        with gr.Accordion('LineArt', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Checkbox(label="Coarse", value=False))
+                        with gr.Accordion('Leres Depth', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Checkbox(label="Boost", value=False))
+                            settings.append(gr.Slider(label="Near threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.0))
+                            settings.append(gr.Slider(label="Background threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.0))
+                        with gr.Accordion('MediaPipe Face', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Slider(label="Max faces", minimum=1, maximum=10, step=1, value=1))
+                            settings.append(gr.Slider(label="Min confidence", minimum=0.0, maximum=1.0, step=0.01, value=0.5))
+                        with gr.Accordion('Canny', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Slider(label="Low threshold", minimum=0, maximum=1000, step=1, value=100))
+                            settings.append(gr.Slider(label="High threshold", minimum=0, maximum=1000, step=1, value=200))
+                        with gr.Accordion('DWPose', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Radio(label="Model", choices=['Tiny', 'Medium', 'Large'], value='Tiny'))
+                            settings.append(gr.Slider(label="Min confidence", minimum=0.0, maximum=1.0, step=0.01, value=0.3))
+                        with gr.Accordion('SegmentAnything', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Radio(label="Model", choices=['Base', 'Large'], value='Base'))
+                        with gr.Accordion('Edge', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Checkbox(label="Parameter free", value=True))
+                            settings.append(gr.Radio(label="Mode", choices=['edge', 'gradient'], value='edge'))
+                        with gr.Accordion('Zoe Depth', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Checkbox(label="Gamma corrected", value=False))
+                        with gr.Accordion('Marigold Depth', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Dropdown(label="Color map", choices=['None'] + plt.colormaps(), value='None'))
+                            settings.append(gr.Slider(label="Denoising steps", minimum=1, maximum=99, step=1, value=10))
+                            settings.append(gr.Slider(label="Ensemble size", minimum=1, maximum=99, step=1, value=10))
+                        with gr.Accordion('Depth Anything', open=True, elem_classes=['processor-settings']):
+                            settings.append(gr.Dropdown(label="Color map", choices=['none'] + masking.COLORMAP, value='inferno'))
+                        for setting in settings:
+                            setting.change(fn=processors.update_settings, inputs=settings, outputs=[])
 
             with gr.Row(elem_id="control_script_container"):
                 input_script_args = scripts.scripts_current.setup_ui(parent='control', accordion=True)
@@ -501,6 +503,8 @@ def create_ui(_blocks: gr.Blocks=None):
                 resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after,
                 resize_mode_mask, resize_name_mask, width_mask, height_mask, scale_by_mask, selected_scale_tab_mask,
                 denoising_strength, batch_count, batch_size,
+                enable_hr, hr_sampler_index, hr_denoising_strength, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps,
+                refiner_start, refiner_prompt, refiner_negative,
                 video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate,
             ]
             output_fields = [

modules/ui_extra_networks.py

@@ -447,10 +447,12 @@ def get_pages(title=None):
 class ExtraNetworksUi:
     def __init__(self):
         self.tabname: str = None
-        self.pages: list(str) = None
+        self.pages: list[str] = None
         self.visible: gr.State = None
         self.state: gr.Textbox = None
         self.details: gr.Group = None
+        self.details_tabs: gr.Group = None
+        self.details_text: gr.Group = None
         self.tabs: gr.Tabs = None
         self.gallery: gr.Gallery = None
         self.description: gr.Textbox = None
@@ -478,7 +480,7 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
     ui.pages = []
     ui.state = gr.Textbox('{}', elem_id=f"{tabname}_extra_state", visible=False)
     ui.visible = gr.State(value=False) # pylint: disable=abstract-class-instantiated
-    ui.details = gr.Group(elem_id=f"{tabname}_extra_details", visible=False)
+    ui.details = gr.Group(elem_id=f"{tabname}_extra_details", elem_classes=["extra-details"], visible=False)
     ui.tabs = gr.Tabs(elem_id=f"{tabname}_extra_tabs")
     ui.button_details = gr.Button('Details', elem_id=f"{tabname}_extra_details_btn", visible=False)
     state = {}
@@ -533,26 +535,39 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
                 with gr.Row():
                     btn_save_img = gr.Button('Replace', elem_classes=['small-button'])
                     btn_delete_img = gr.Button('Delete', elem_classes=['small-button'])
-        with gr.Tabs():
-            with gr.Tab('Description'):
-                desc = gr.Textbox('', show_label=False, lines=8, placeholder="Extra network description...")
-                ui.details_components.append(desc)
-                with gr.Row():
-                    btn_save_desc = gr.Button('Save', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_save_desc')
-                    btn_delete_desc = gr.Button('Delete', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_delete_desc')
-                    btn_close_desc = gr.Button('Close', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_close_desc')
-                    btn_close_desc.click(fn=lambda: gr.update(visible=False), _js='refeshDetailsEN', inputs=[], outputs=[ui.details])
-            with gr.Tab('Model metadata'):
-                info = gr.JSON({}, show_label=False)
-                ui.details_components.append(info)
-                with gr.Row():
-                    btn_save_info = gr.Button('Save', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_save_info')
-                    btn_delete_info = gr.Button('Delete', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_delete_info')
-                    btn_close_info = gr.Button('Close', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_close_info')
-                    btn_close_info.click(fn=lambda: gr.update(visible=False), _js='refeshDetailsEN', inputs=[], outputs=[ui.details])
-            with gr.Tab('Embedded metadata'):
-                meta = gr.JSON({}, show_label=False)
-                ui.details_components.append(meta)
+        with gr.Group(elem_id=f"{tabname}_extra_details_tabs", visible=False) as ui.details_tabs:
+            with gr.Tabs():
+                with gr.Tab('Description', elem_classes=['extra-details-tabs']):
+                    desc = gr.Textbox('', show_label=False, lines=8, placeholder="Extra network description...")
+                    ui.details_components.append(desc)
+                    with gr.Row():
+                        btn_save_desc = gr.Button('Save', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_save_desc')
+                        btn_delete_desc = gr.Button('Delete', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_delete_desc')
+                        btn_close_desc = gr.Button('Close', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_close_desc')
+                        btn_close_desc.click(fn=lambda: gr.update(visible=False), _js='refeshDetailsEN', inputs=[], outputs=[ui.details])
+                with gr.Tab('Model metadata', elem_classes=['extra-details-tabs']):
+                    info = gr.JSON({}, show_label=False)
+                    ui.details_components.append(info)
+                    with gr.Row():
+                        btn_save_info = gr.Button('Save', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_save_info')
+                        btn_delete_info = gr.Button('Delete', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_delete_info')
+                        btn_close_info = gr.Button('Close', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_close_info')
+                        btn_close_info.click(fn=lambda: gr.update(visible=False), _js='refeshDetailsEN', inputs=[], outputs=[ui.details])
+                with gr.Tab('Embedded metadata', elem_classes=['extra-details-tabs']):
+                    meta = gr.JSON({}, show_label=False)
+                    ui.details_components.append(meta)
+        with gr.Group(elem_id=f"{tabname}_extra_details_text", elem_classes=["extra-details-text"], visible=False) as ui.details_text:
+            description = gr.Textbox(label='Description', lines=1, placeholder="Style description...")
+            prompt = gr.Textbox(label='Prompt', lines=2, placeholder="Prompt...")
+            negative = gr.Textbox(label='Negative prompt', lines=2, placeholder="Negative prompt...")
+            extra = gr.Textbox(label='Parameters', lines=2, placeholder="Generation parameters overrides...")
+            wildcards = gr.Textbox(label='Wildcards', lines=2, placeholder="Wildcard prompt replacements...")
+            ui.details_components += [description, prompt, negative, extra, wildcards]
+            with gr.Row():
+                btn_save_style = gr.Button('Save', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_save_style')
+                btn_delete_style = gr.Button('Delete', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_delete_style')
+                btn_close_style = gr.Button('Close', elem_classes=['small-button'], elem_id=f'{tabname}_extra_details_close_style')
+                btn_close_style.click(fn=lambda: gr.update(visible=False), _js='refeshDetailsEN', inputs=[], outputs=[ui.details])
 
     with ui.tabs:
         def ui_tab_change(page):
@@ -570,7 +585,7 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
         ui.button_close = ToolButton(symbols.close, elem_id=f"{tabname}_extra_close", visible=True)
         ui.button_model = ToolButton(symbols.refine, elem_id=f"{tabname}_extra_model", visible=True)
         ui.search = gr.Textbox('', show_label=False, elem_id=f"{tabname}_extra_search", placeholder="Search...", elem_classes="textbox", lines=2, container=False)
-        ui.description = gr.Textbox('', show_label=False, elem_id=f"{tabname}_description", elem_classes="textbox", lines=2, interactive=False, container=False)
+        ui.description = gr.Textbox('', show_label=False, elem_id=f"{tabname}_description", elem_classes=["textbox", "extra-description"], lines=2, interactive=False, container=False)
 
         if ui.tabname == 'txt2img': # refresh only once
             global refresh_time # pylint: disable=global-statement
@@ -639,10 +654,7 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
     def fn_delete_desc(desc):
         if ui.last_item is None:
             return desc
-        if hasattr(ui.last_item, 'type') and ui.last_item.type == 'Style':
-            fn = os.path.splitext(ui.last_item.filename)[0] + '.json'
-        else:
-            fn = os.path.splitext(ui.last_item.filename)[0] + '.txt'
+        fn = os.path.splitext(ui.last_item.filename)[0] + '.txt'
         if os.path.exists(fn):
             shared.log.debug(f'Extra network delete desc: item={ui.last_item.name} filename="{fn}"')
             os.remove(fn)
@@ -665,14 +677,39 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
             return ''
         return info
 
+    def fn_save_style(info, description, prompt, negative, extra, wildcards):
+        if not isinstance(info, dict) or isinstance(info, list):
+            shared.log.warning(f'Extra network save style skip: item={ui.last_item.name} not a dict: {type(info)}')
+            return info
+        if ui.last_item is None:
+            return info
+        fn = os.path.splitext(ui.last_item.filename)[0] + '.json'
+        if hasattr(ui.last_item, 'type') and ui.last_item.type == 'Style':
+            info.update(**{ 'description': description, 'prompt': prompt, 'negative': negative, 'extra': extra, 'wildcards': wildcards })
+            shared.writefile(info, fn, silent=True)
+            shared.log.debug(f'Extra network save style: item={ui.last_item.name} filename="{fn}"')
+        return info
+
+    def fn_delete_style(info):
+        if ui.last_item is None:
+            return info
+        fn = os.path.splitext(ui.last_item.filename)[0] + '.json'
+        if os.path.exists(fn):
+            shared.log.debug(f'Extra network delete style: item={ui.last_item.name} filename="{fn}"')
+            os.remove(fn)
+            return {}
+        return info
+
     btn_save_img.click(fn=fn_save_img, _js='closeDetailsEN', inputs=[img], outputs=[img])
     btn_delete_img.click(fn=fn_delete_img, _js='closeDetailsEN', inputs=[img], outputs=[img])
     btn_save_desc.click(fn=fn_save_desc, _js='closeDetailsEN', inputs=[desc], outputs=[desc])
     btn_delete_desc.click(fn=fn_delete_desc, _js='closeDetailsEN', inputs=[desc], outputs=[desc])
     btn_save_info.click(fn=fn_save_info, _js='closeDetailsEN', inputs=[info], outputs=[info])
     btn_delete_info.click(fn=fn_delete_info, _js='closeDetailsEN', inputs=[info], outputs=[info])
+    btn_save_style.click(fn=fn_save_style, _js='closeDetailsEN', inputs=[info, description, prompt, negative, extra, wildcards], outputs=[info])
+    btn_delete_style.click(fn=fn_delete_style, _js='closeDetailsEN', inputs=[info], outputs=[info])
 
-    def show_details(text, img, desc, info, meta, params):
+    def show_details(text, img, desc, info, meta, description, prompt, negative, parameters, wildcards, params, _dummy1=None, _dummy2=None):
         page, item = get_item(state, params)
         if item is not None and hasattr(item, 'name'):
             stat = os.stat(item.filename) if os.path.exists(item.filename) else None
@@ -730,12 +767,17 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
                     <tr><td>Comment</td><td>{meta.get('ss_training_comment', 'N/A')}</td></tr>
                 '''
             if page.title == 'Style':
+                description = item.description
+                prompt = item.prompt
+                negative = item.negative
+                parameters = item.extra
+                wildcards = item.wildcards
                 style = f'''
                     <tr><td>Name</td><td>{item.name}</td></tr>
                     <tr><td>Description</td><td>{item.description}</td></tr>
                     <tr><td>Preview Embedded</td><td>{item.preview.startswith('data:')}</td></tr>
                 '''
-                desc = f'Name: {os.path.basename(item.name)}\nDescription: {item.description}\nPrompt: {item.prompt}\nNegative: {item.negative}\nExtra: {item.extra}\n'
+                # desc = f'Name: {os.path.basename(item.name)}\nDescription: {item.description}\nPrompt: {item.prompt}\nNegative: {item.negative}\nExtra: {item.extra}\n'
             text = f'''
                 <h2 style="border-bottom: 1px solid var(--button-primary-border-color); margin: 0em 0px 1em 0 !important">{item.name}</h2>
                 <table style="width: 100%; line-height: 1.3em;"><tbody>
@@ -752,7 +794,21 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
                 </tbody></table>
                 {note}
             '''
-        return [text, img, desc, info, meta, gr.update(visible=item is not None)]
+        return [
+            text, # gr.html
+            img, # gr.image
+            desc, # gr.textbox
+            info, # gr.json
+            meta, # gr.json
+            description, # gr.textbox
+            prompt, # gr.textbox
+            negative, # gr.textbox
+            parameters, # gr.textbox
+            wildcards, # gr.textbox
+            gr.update(visible=item is not None), # details ui visible
+            gr.update(visible=page is not None and page.title != 'Style'), # details ui tabs visible
+            gr.update(visible=page is not None and page.title == 'Style'), # details ui text visible
+        ]
 
     def ui_refresh_click(title):
         pages = []
@@ -796,7 +852,7 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
         else:
             prompt = ''
         params = generation_parameters_copypaste.parse_generation_parameters(prompt)
-        res = show_details(text=None, img=None, desc=None, info=None, meta=None, params=params)
+        res = show_details(text=None, img=None, desc=None, info=None, meta=None, parameters=None, description=None, prompt=None, negative=None, wildcards=None, params=params)
         return res
 
     def ui_quicksave_click(name):
@@ -836,7 +892,7 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
     ui.button_scan.click(fn=ui_scan_click, _js='getENActivePage', inputs=[ui.search], outputs=ui.pages)
     ui.button_save.click(fn=ui_save_click, inputs=[], outputs=ui.details_components + [ui.details])
     ui.button_quicksave.click(fn=ui_quicksave_click, _js="() => prompt('Prompt name', '')", inputs=[ui.search], outputs=[])
-    ui.button_details.click(show_details, _js="getCardDetails", inputs=ui.details_components + [dummy], outputs=ui.details_components + [ui.details])
+    ui.button_details.click(show_details, _js="getCardDetails", inputs=ui.details_components + [dummy, dummy, dummy], outputs=ui.details_components + [ui.details, ui.details_tabs, ui.details_text])
     ui.state.change(state_change, inputs=[ui.state], outputs=[])
     return ui
 

modules/ui_extra_networks_styles.py

@@ -13,7 +13,7 @@ class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage):
 
     def parse_desc(self, desc):
         lines = desc.strip().split("\n")
-        params = { 'name': '', 'description': '', 'prompt': '', 'negative': '', 'extra': ''}
+        params = { 'name': '', 'description': '', 'prompt': '', 'negative': '', 'extra': '', 'wildcards': ''}
         found = ''
         for line in lines:
             line = line.strip()
@@ -32,6 +32,9 @@ class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage):
             elif line.lower().startswith('extra:'):
                 found = 'extra'
                 params['extra'] = line[6:].strip()
+            elif line.lower().startswith('wildcards:'):
+                found = 'wildcards'
+                params['wildcards'] = line[10:].strip()
             elif found != '':
                 params[found] += '\n' + line
         if params['name'] == '':
@@ -53,10 +56,11 @@ class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage):
             "title": name,
             "filename": fn,
             "preview": self.find_preview(name),
-            "description": '',
+            "description": params.get('Description', ''),
             "prompt": params.get('Prompt', ''),
             "negative": params.get('Negative prompt', ''),
-            "extra": '',
+            "extra": params.get('Extra', ''),
+            "wildcards": params.get('Wildcards', ''),
             "local_preview": f"{name}.{shared.opts.samples_format}",
         }
         return item
@@ -82,6 +86,7 @@ class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage):
                 "prompt": getattr(style, 'prompt', ''),
                 "negative": getattr(style, 'negative_prompt', ''),
                 "extra": getattr(style, 'extra', ''),
+                "wildcards": getattr(style, 'wildcards', ''),
                 "local_preview": f"{fn}.{shared.opts.samples_format}",
                 "onclick": '"' + html.escape(f"""return selectStyle({json.dumps(name)})""") + '"',
                 "mtime": getattr(style, 'mtime', 0),

modules/ui_img2img.py

@@ -58,10 +58,10 @@ def create_ui():
                     with gr.Row(variant="compact", elem_id=f"img2img_copy_to_{tab_name}"):
                         for title, name in zip(['➠ Image', '➠ Sketch', '➠ Inpaint', '➠ Composite'], ['img2img', 'sketch', 'inpaint', 'inpaint_sketch']):
                             if name == tab_name:
-                                gr.Button(title, interactive=False)
+                                gr.Button(title, elem_id=f'copy_to_{name}', interactive=False)
                                 copy_image_destinations[name] = elem
                                 continue
-                            button = gr.Button(title)
+                            button = gr.Button(title, elem_id=f'copy_to_{name}')
                             copy_image_buttons.append((button, name, elem))
 
                 with gr.Tabs(elem_id="mode_img2img"):
@@ -134,7 +134,7 @@ def create_ui():
                     hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio, = ui_sections.create_correction_inputs('img2img')
 
                     # with gr.Group(elem_id="inpaint_controls", visible=False) as inpaint_controls:
-                    with gr.Accordion(open=True, label="Mask", elem_classes=["small-accordion"], elem_id="img2img_mask_group") as inpaint_controls:
+                    with gr.Accordion(open=False, label="Mask", elem_classes=["small-accordion"], elem_id="img2img_mask_group") as inpaint_controls:
                         with gr.Row():
                             mask_blur = gr.Slider(label='Blur', minimum=0, maximum=64, step=1, value=4, elem_id="img2img_mask_blur")
                             inpaint_full_res_padding = gr.Slider(label='Padding', minimum=0, maximum=256, step=4, value=32, elem_id="img2img_inpaint_full_res_padding")

modules/ui_interrogate.py

@@ -1,231 +0,0 @@
-import os
-import gradio as gr
-import torch
-from PIL import Image
-import modules.generation_parameters_copypaste as parameters_copypaste
-from modules import devices, lowvram, shared, paths, ui_common
-
-
-ci = None
-low_vram = False
-
-
-class BatchWriter:
-    def __init__(self, folder):
-        self.folder = folder
-        self.csv, self.file = None, None
-
-    def add(self, file, prompt):
-        txt_file = os.path.splitext(file)[0] + ".txt"
-        with open(os.path.join(self.folder, txt_file), 'w', encoding='utf-8') as f:
-            f.write(prompt)
-
-    def close(self):
-        if self.file is not None:
-            self.file.close()
-
-
-def get_models():
-    import open_clip
-    return ['/'.join(x) for x in open_clip.list_pretrained()]
-
-
-def load_interrogator(clip_model_name):
-    from clip_interrogator import Config, Interrogator
-    global ci # pylint: disable=global-statement
-    if ci is None:
-        config = Config(device=devices.get_optimal_device(), cache_path=os.path.join(paths.models_path, 'Interrogator'), clip_model_name=clip_model_name, quiet=True)
-        if low_vram:
-            config.apply_low_vram_defaults()
-        shared.log.info(f'Interrogate load: config={config}')
-        ci = Interrogator(config)
-    elif clip_model_name != ci.config.clip_model_name:
-        ci.config.clip_model_name = clip_model_name
-        shared.log.info(f'Interrogate load: config={ci.config}')
-        ci.load_clip_model()
-
-
-def unload():
-    if ci is not None:
-        shared.log.debug('Interrogate offload')
-        ci.caption_model = ci.caption_model.to(devices.cpu)
-        ci.clip_model = ci.clip_model.to(devices.cpu)
-        ci.caption_offloaded = True
-        ci.clip_offloaded = True
-        devices.torch_gc()
-
-
-def interrogate(image, mode, caption=None):
-    shared.log.info(f'Interrogate: image={image} mode={mode} config={ci.config}')
-    if mode == 'best':
-        prompt = ci.interrogate(image, caption=caption)
-    elif mode == 'caption':
-        prompt = ci.generate_caption(image) if caption is None else caption
-    elif mode == 'classic':
-        prompt = ci.interrogate_classic(image, caption=caption)
-    elif mode == 'fast':
-        prompt = ci.interrogate_fast(image, caption=caption)
-    elif mode == 'negative':
-        prompt = ci.interrogate_negative(image)
-    else:
-        raise RuntimeError(f"Unknown mode {mode}")
-    return prompt
-
-
-def interrogate_image(image, model, mode):
-    shared.state.begin()
-    shared.state.job = 'interrogate'
-    try:
-        if shared.backend == shared.Backend.ORIGINAL and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram):
-            lowvram.send_everything_to_cpu()
-            devices.torch_gc()
-        load_interrogator(model)
-        image = image.convert('RGB')
-        shared.log.info(f'Interrogate: image={image} mode={mode} config={ci.config}')
-        prompt = interrogate(image, mode)
-    except Exception as e:
-        prompt = f"Exception {type(e)}"
-        shared.log.error(f'Interrogate: {e}')
-    shared.state.end()
-    return prompt
-
-
-def interrogate_batch(batch_files, batch_folder, batch_str, model, mode, write):
-    files = []
-    if batch_files is not None:
-        files += [f.name for f in batch_files]
-    if batch_folder is not None:
-        files += [f.name for f in batch_folder]
-    if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str):
-        files += [os.path.join(batch_str, f) for f in os.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))]
-    if len(files) == 0:
-        shared.log.error('Interrogate batch no images')
-        return ''
-    shared.state.begin()
-    shared.state.job = 'batch interrogate'
-    prompts = []
-    try:
-        if shared.backend == shared.Backend.ORIGINAL and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram):
-            lowvram.send_everything_to_cpu()
-            devices.torch_gc()
-        load_interrogator(model)
-        shared.log.info(f'Interrogate batch: images={len(files)} mode={mode} config={ci.config}')
-        captions = []
-        # first pass: generate captions
-        for file in files:
-            caption = ""
-            try:
-                if shared.state.interrupted:
-                    break
-                image = Image.open(file).convert('RGB')
-                caption = ci.generate_caption(image)
-            except Exception as e:
-                shared.log.error(f'Interrogate caption: {e}')
-            finally:
-                captions.append(caption)
-        # second pass: interrogate
-        if write:
-            writer = BatchWriter(os.path.dirname(files[0]))
-        for idx, file in enumerate(files):
-            try:
-                if shared.state.interrupted:
-                    break
-                image = Image.open(file).convert('RGB')
-                prompt = interrogate(image, mode, caption=captions[idx])
-                prompts.append(prompt)
-                if write:
-                    writer.add(file, prompt)
-            except OSError as e:
-                shared.log.error(f'Interrogate batch: {e}')
-        if write:
-            writer.close()
-        ci.config.quiet = False
-        unload()
-    except Exception as e:
-        shared.log.error(f'Interrogate batch: {e}')
-    shared.state.end()
-    return '\n\n'.join(prompts)
-
-
-def analyze_image(image, model):
-    load_interrogator(model)
-    image = image.convert('RGB')
-    image_features = ci.image_to_features(image)
-    top_mediums = ci.mediums.rank(image_features, 5)
-    top_artists = ci.artists.rank(image_features, 5)
-    top_movements = ci.movements.rank(image_features, 5)
-    top_trendings = ci.trendings.rank(image_features, 5)
-    top_flavors = ci.flavors.rank(image_features, 5)
-    medium_ranks = dict(zip(top_mediums, ci.similarities(image_features, top_mediums)))
-    artist_ranks = dict(zip(top_artists, ci.similarities(image_features, top_artists)))
-    movement_ranks = dict(zip(top_movements, ci.similarities(image_features, top_movements)))
-    trending_ranks = dict(zip(top_trendings, ci.similarities(image_features, top_trendings)))
-    flavor_ranks = dict(zip(top_flavors, ci.similarities(image_features, top_flavors)))
-    return medium_ranks, artist_ranks, movement_ranks, trending_ranks, flavor_ranks
-
-
-def create_ui():
-    global low_vram # pylint: disable=global-statement
-    low_vram = shared.cmd_opts.lowvram or shared.cmd_opts.medvram
-    if not low_vram and torch.cuda.is_available():
-        device = devices.get_optimal_device()
-        vram_total = torch.cuda.get_device_properties(device).total_memory
-        if vram_total <= 12*1024*1024*1024:
-            low_vram = True
-    with gr.Row(elem_id="interrogate_tab"):
-        with gr.Column():
-            with gr.Tab("Image"):
-                with gr.Row():
-                    image = gr.Image(type='pil', label="Image")
-                with gr.Row():
-                    prompt = gr.Textbox(label="Prompt", lines=3)
-                with gr.Row():
-                    medium = gr.Label(label="Medium", num_top_classes=5)
-                    artist = gr.Label(label="Artist", num_top_classes=5)
-                    movement = gr.Label(label="Movement", num_top_classes=5)
-                    trending = gr.Label(label="Trending", num_top_classes=5)
-                    flavor = gr.Label(label="Flavor", num_top_classes=5)
-                with gr.Row():
-                    clip_model = gr.Dropdown([], value='ViT-L-14/openai', label='CLIP Model')
-                    ui_common.create_refresh_button(clip_model, get_models, lambda: {"choices": get_models()}, 'refresh_interrogate_models')
-                    mode = gr.Radio(['best', 'fast', 'classic', 'caption', 'negative'], label='Mode', value='best')
-                with gr.Row():
-                    btn_interrogate_img = gr.Button("Interrogate", variant='primary')
-                    btn_analyze_img = gr.Button("Analyze", variant='primary')
-                    btn_unload = gr.Button("Unload")
-                with gr.Row():
-                    buttons = parameters_copypaste.create_buttons(["txt2img", "img2img", "extras", "control"])
-                for tabname, button in buttons.items():
-                    parameters_copypaste.register_paste_params_button(parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=prompt, source_image_component=image,))
-                btn_interrogate_img.click(interrogate_image, inputs=[image, clip_model, mode], outputs=prompt)
-                btn_analyze_img.click(analyze_image, inputs=[image, clip_model], outputs=[medium, artist, movement, trending, flavor])
-                btn_unload.click(unload)
-            with gr.Tab("Batch"):
-                with gr.Row():
-                    batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100)
-                with gr.Row():
-                    batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100)
-                with gr.Row():
-                    batch_str = gr.Text(label="Folder", value="", interactive=True)
-                with gr.Row():
-                    batch = gr.Text(label="Prompts", lines=10)
-                with gr.Row():
-                    write = gr.Checkbox(label='Write prompts to files', value=False)
-                with gr.Row():
-                    clip_model = gr.Dropdown([], value='ViT-L-14/openai', label='CLIP Model')
-                    ui_common.create_refresh_button(clip_model, get_models, lambda: {"choices": get_models()}, 'refresh_interrogate_models')
-                with gr.Row():
-                    btn_interrogate_batch = gr.Button("Interrogate", variant='primary')
-                btn_interrogate_batch.click(interrogate_batch, inputs=[batch_files, batch_folder, batch_str, clip_model, mode, write], outputs=[batch])
-            with gr.Tab("VQA"):
-                from modules import vqa
-                with gr.Row():
-                    vqa_image = gr.Image(type='pil', label="Image")
-                with gr.Row():
-                    vqa_question = gr.Textbox(label="Question")
-                with gr.Row():
-                    vqa_answer = gr.Textbox(label="Answer", lines=3)
-                with gr.Row():
-                    vqa_model = gr.Dropdown(list(vqa.MODELS), value='None', label='VQA Model')
-                    vqa_submit = gr.Button("Interrogate", variant='primary')
-                vqa_submit.click(vqa.interrogate, inputs=[vqa_question, vqa_image, vqa_model], outputs=[vqa_answer])

modules/ui_javascript.py

@@ -60,6 +60,8 @@ def html_css(is_builtin: bool):
     theme_name = modules.shared.cmd_opts.theme or modules.shared.opts.gradio_theme or ''
     if theme_name == 'default':
         theme_name = 'black-teal'
+    if theme_name == 'modern' or theme_name == 'modern/default':
+        theme_name = 'modern/sdxl_alpha'
     if theme_name.startswith('modern/'):
         theme_name = theme_name[7:]
         theme_folder = next((e.path for e in modules.extensions.extensions if e.name == 'sdnext-ui-ux'), None)

modules/ui_postprocessing.py

@@ -1,18 +1,18 @@
 import json
 import gradio as gr
-from modules import scripts, shared, ui_common, postprocessing, call_queue
+from modules import scripts, shared, ui_common, postprocessing, call_queue, interrogate
 import modules.generation_parameters_copypaste as parameters_copypaste
 from modules.call_queue import wrap_gradio_gpu_call, wrap_queued_call, wrap_gradio_call # pylint: disable=unused-import
 from modules.extras import run_pnginfo
 from modules.ui_common import infotext_to_html
 
 
-def wrap_pnginfo(image):
+def submit_info(image):
     _, geninfo, info = run_pnginfo(image)
     return infotext_to_html(geninfo), info, geninfo
 
 
-def submit_click(tab_index, extras_image, image_batch, extras_batch_input_dir, extras_batch_output_dir, show_extras_results, save_output, *script_inputs):
+def submit_process(tab_index, extras_image, image_batch, extras_batch_input_dir, extras_batch_output_dir, show_extras_results, save_output, *script_inputs):
     result_images, geninfo, js_info = postprocessing.run_postprocessing(tab_index, extras_image, image_batch, extras_batch_input_dir, extras_batch_output_dir, show_extras_results, *script_inputs, save_output=save_output)
     return result_images, geninfo, json.dumps(js_info), ''
 
@@ -22,18 +22,72 @@ def create_ui():
     with gr.Row(equal_height=False, variant='compact', elem_classes="extras"):
         with gr.Column(variant='compact'):
             with gr.Tabs(elem_id="mode_extras"):
-                with gr.TabItem('Single Image', id="single_image", elem_id="extras_single_tab") as tab_single:
-                    extras_image = gr.Image(label="Source", source="upload", interactive=True, type="pil", elem_id="extras_image")
-                with gr.TabItem('Process Batch', id="batch_process", elem_id="extras_batch_process_tab") as tab_batch:
+                with gr.Tab('Process Image', id="single_image", elem_id="extras_single_tab") as tab_single:
+                    with gr.Row():
+                        extras_image = gr.Image(label="Source", source="upload", interactive=True, type="pil", elem_id="extras_image")
+                    with gr.Row(elem_id='copy_buttons_process'):
+                        copy_process_buttons = parameters_copypaste.create_buttons(["txt2img", "img2img", "inpaint", "control"])
+                with gr.Tab('Process Batch', id="batch_process", elem_id="extras_batch_process_tab") as tab_batch:
                     image_batch = gr.Files(label="Batch process", interactive=True, elem_id="extras_image_batch")
-                with gr.TabItem('Process Folder', id="batch_from_directory", elem_id="extras_batch_directory_tab") as tab_batch_dir:
+                with gr.Tab('Process Folder', id="batch_from_directory", elem_id="extras_batch_directory_tab") as tab_batch_dir:
                     extras_batch_input_dir = gr.Textbox(label="Input directory", **shared.hide_dirs, placeholder="A directory on the same machine where the server is running.", elem_id="extras_batch_input_dir")
                     extras_batch_output_dir = gr.Textbox(label="Output directory", **shared.hide_dirs, placeholder="Leave blank to save images to the default path.", elem_id="extras_batch_output_dir")
                     show_extras_results = gr.Checkbox(label='Show result images', value=True, elem_id="extras_show_extras_results")
-            with gr.Row():
-                buttons = parameters_copypaste.create_buttons(["txt2img", "img2img", "inpaint", "control"])
-            with gr.Row():
-                save_output = gr.Checkbox(label='Save output', value=True, elem_id="extras_save_output")
+
+                with gr.Tab("Interrogate Image"):
+                    with gr.Row():
+                        image = gr.Image(type='pil', label="Image")
+                    with gr.Row():
+                        prompt = gr.Textbox(label="Prompt", lines=3)
+                    with gr.Row(elem_id="interrogate_labels"):
+                        medium = gr.Label(elem_id="interrogate_label_medium", label="Medium", num_top_classes=5)
+                        artist = gr.Label(elem_id="interrogate_label_artist", label="Artist", num_top_classes=5)
+                        movement = gr.Label(elem_id="interrogate_label_movement", label="Movement", num_top_classes=5)
+                        trending = gr.Label(elem_id="interrogate_label_trending", label="Trending", num_top_classes=5)
+                        flavor = gr.Label(elem_id="interrogate_label_flavor", label="Flavor", num_top_classes=5)
+                    with gr.Row():
+                        clip_model = gr.Dropdown([], value='ViT-L-14/openai', label='CLIP Model')
+                        ui_common.create_refresh_button(clip_model, interrogate.get_clip_models, lambda: {"choices": interrogate.get_clip_models()}, 'refresh_interrogate_models')
+                        mode = gr.Radio(['best', 'fast', 'classic', 'caption', 'negative'], label='Mode', value='best')
+                    with gr.Row(elem_id='interrogate_buttons_image'):
+                        btn_interrogate_img = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
+                        btn_analyze_img = gr.Button("Analyze", elem_id="interrogate_btn_analyze", variant='primary')
+                        btn_unload = gr.Button("Unload", elem_id="interrogate_btn_unload")
+                    with gr.Row(elem_id='copy_buttons_interrogate'):
+                        copy_interrogate_buttons = parameters_copypaste.create_buttons(["txt2img", "img2img", "extras", "control"])
+                    btn_interrogate_img.click(interrogate.interrogate_image, inputs=[image, clip_model, mode], outputs=prompt)
+                    btn_analyze_img.click(interrogate.analyze_image, inputs=[image, clip_model], outputs=[medium, artist, movement, trending, flavor])
+                    btn_unload.click(interrogate.unload_clip_model)
+                with gr.Tab("Interrogate Batch"):
+                    with gr.Row():
+                        batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], type='file', interactive=True, height=100)
+                    with gr.Row():
+                        batch_folder = gr.File(label="Folder", show_label=True, file_count='directory', file_types=['image'], type='file', interactive=True, height=100)
+                    with gr.Row():
+                        batch_str = gr.Text(label="Folder", value="", interactive=True)
+                    with gr.Row():
+                        batch = gr.Text(label="Prompts", lines=10)
+                    with gr.Row():
+                        clip_model = gr.Dropdown([], value='ViT-L-14/openai', label='CLIP Model')
+                        ui_common.create_refresh_button(clip_model, interrogate.get_clip_models, lambda: {"choices": interrogate.get_clip_models()}, 'refresh_interrogate_models')
+                    with gr.Row(elem_id='interrogate_buttons_batch'):
+                        btn_interrogate_batch = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
+                with gr.Tab("Visual Query"):
+                    from modules import vqa
+                    with gr.Row():
+                        vqa_image = gr.Image(type='pil', label="Image")
+                    with gr.Row():
+                        vqa_question = gr.Textbox(label="Question", placeholder="Descirbe the image")
+                    with gr.Row():
+                        vqa_answer = gr.Textbox(label="Answer", lines=3)
+                    with gr.Row(elem_id='interrogate_buttons_query'):
+                        vqa_model = gr.Dropdown(list(vqa.MODELS), value='Moondream 2', label='VQA Model')
+                        vqa_submit = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
+                    vqa_submit.click(vqa.interrogate, inputs=[vqa_question, vqa_image, vqa_model], outputs=[vqa_answer])
+
+                with gr.Row():
+                    save_output = gr.Checkbox(label='Save output', value=True, elem_id="extras_save_output")
+
             script_inputs = scripts.scripts_postproc.setup_ui()
         with gr.Column():
             id_part = 'extras'
@@ -47,20 +101,23 @@ def create_ui():
             gr.HTML('File metadata')
             exif_info = gr.HTML(elem_id="pnginfo_html_info")
             gen_info = gr.Text(elem_id="pnginfo_gen_info", visible=False)
-        for tabname, button in buttons.items():
+        for tabname, button in copy_process_buttons.items():
             parameters_copypaste.register_paste_params_button(parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=gen_info, source_image_component=extras_image))
+        for tabname, button in copy_interrogate_buttons.items():
+            parameters_copypaste.register_paste_params_button(parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=prompt, source_image_component=image,))
+
 
     tab_single.select(fn=lambda: 0, inputs=[], outputs=[tab_index])
     tab_batch.select(fn=lambda: 1, inputs=[], outputs=[tab_index])
     tab_batch_dir.select(fn=lambda: 2, inputs=[], outputs=[tab_index])
     extras_image.change(
-        fn=wrap_gradio_call(wrap_pnginfo),
+        fn=wrap_gradio_call(submit_info),
         inputs=[extras_image],
         outputs=[html_info_formatted, exif_info, gen_info],
     )
     submit.click(
         _js="submit_postprocessing",
-        fn=call_queue.wrap_gradio_gpu_call(submit_click, extra_outputs=[None, '']),
+        fn=call_queue.wrap_gradio_gpu_call(submit_process, extra_outputs=[None, '']),
         inputs=[
             tab_index,
             extras_image,
@@ -78,6 +135,11 @@ def create_ui():
             html_log,
         ]
     )
+    btn_interrogate_batch.click(
+        fn=interrogate.interrogate_batch,
+        inputs=[batch_files, batch_folder, batch_str, clip_model, mode, save_output],
+        outputs=[batch],
+    )
 
     parameters_copypaste.add_paste_fields("extras", extras_image, None)
 

modules/ui_sections.py

@@ -55,6 +55,34 @@ def create_toprow(is_img2img: bool = False, id_part: str = None):
     return prompt, styles, negative_prompt, submit, button_paste, button_extra, token_counter, token_button, negative_token_counter, negative_token_button
 
 
+def ar_change(ar, width, height):
+    if ar == 'AR':
+        return gr.update(interactive=True), gr.update(interactive=True)
+    try:
+        (w, h) = [float(x) for x in ar.split(':')]
+    except Exception as e:
+        shared.log.warning(f"Invalid aspect ratio: {ar} {e}")
+        return gr.update(interactive=True), gr.update(interactive=True)
+    if w > h:
+        return gr.update(interactive=True, value=width), gr.update(interactive=False, value=int(width * h / w))
+    elif w < h:
+        return gr.update(interactive=False, value=int(height * w / h)), gr.update(interactive=True, value=height)
+    else:
+        return gr.update(interactive=True, value=width), gr.update(interactive=False, value=width)
+
+
+def create_resolution_inputs(tab):
+    width = gr.Slider(minimum=64, maximum=4096, step=8, label="Width", value=512, elem_id=f"{tab}_width")
+    height = gr.Slider(minimum=64, maximum=4096, step=8, label="Height", value=512, elem_id=f"{tab}_height")
+    ar_list = ['AR'] + [x.strip() for x in shared.opts.aspect_ratios.split(',') if x.strip() != '']
+    ar_dropdown = gr.Dropdown(show_label=False, interactive=True, choices=ar_list, value=ar_list[0], elem_id=f"{tab}_ar", elem_classes=["ar-dropdown"])
+    for c in [ar_dropdown, width, height]:
+        c.change(fn=ar_change, inputs=[ar_dropdown, width, height], outputs=[width, height], show_progress=False)
+    res_switch_btn = ToolButton(value=ui_symbols.switch, elem_id=f"{tab}_res_switch_btn", label="Switch dims")
+    res_switch_btn.click(lambda w, h: (h, w), inputs=[width, height], outputs=[width, height], show_progress=False)
+    return width, height
+
+
 def create_interrogate_buttons(tab):
     button_interrogate = gr.Button(ui_symbols.int_clip, elem_id=f"{tab}_interrogate", elem_classes=['interrogate-clip'])
     button_deepbooru = gr.Button(ui_symbols.int_blip, elem_id=f"{tab}_deepbooru", elem_classes=['interrogate-blip'])
@@ -74,8 +102,6 @@ def create_batch_inputs(tab):
         with gr.Row(elem_id=f"{tab}_row_batch"):
             batch_count = gr.Slider(minimum=1, step=1, label='Batch count', value=1, elem_id=f"{tab}_batch_count")
             batch_size = gr.Slider(minimum=1, maximum=32, step=1, label='Batch size', value=1, elem_id=f"{tab}_batch_size")
-            batch_switch_btn = ToolButton(value=ui_symbols.switch, elem_id=f"{tab}_batch_switch_btn", label="Switch dims")
-            batch_switch_btn.click(lambda w, h: (h, w), inputs=[batch_count, batch_size], outputs=[batch_count, batch_size], show_progress=False)
     return batch_count, batch_size
 
 
@@ -241,6 +267,10 @@ def create_resize_inputs(tab, images, scale_visible=True, mode=None, accordion=T
                                 with gr.Row():
                                     width = gr.Slider(minimum=64, maximum=8192, step=8, label="Width", value=512, elem_id=f"{tab}_width")
                                     height = gr.Slider(minimum=64, maximum=8192, step=8, label="Height", value=512, elem_id=f"{tab}_height")
+                                    ar_list = ['AR'] + [x.strip() for x in shared.opts.aspect_ratios.split(',') if x.strip() != '']
+                                    ar_dropdown = gr.Dropdown(show_label=False, interactive=True, choices=ar_list, value=ar_list[0], elem_id=f"{tab}_ar", elem_classes=["ar-dropdown"])
+                                    for c in [ar_dropdown, width, height]:
+                                        c.change(fn=ar_change, inputs=[ar_dropdown, width, height], outputs=[width, height], show_progress=False)
                                     res_switch_btn = ToolButton(value=ui_symbols.switch, elem_id=f"{tab}_res_switch_btn")
                                     res_switch_btn.click(lambda w, h: (h, w), inputs=[width, height], outputs=[width, height], show_progress=False)
                                     detect_image_size_btn = ToolButton(value=ui_symbols.detect, elem_id=f"{tab}_detect_image_size_btn")

modules/ui_txt2img.py

@@ -1,7 +1,7 @@
 import gradio as gr
 from modules.call_queue import wrap_gradio_gpu_call, wrap_queued_call
-from modules import timer, shared, ui_common, ui_symbols, ui_sections, generation_parameters_copypaste
-from modules.ui_components import ToolButton
+from modules import timer, shared, ui_common, ui_sections, generation_parameters_copypaste
+from modules.ui_components import ToolButton # pylint: disable=unused-import
 
 
 def calc_resolution_hires(width, height, hr_scale, hr_resize_x, hr_resize_y, hr_upscaler):
@@ -35,10 +35,7 @@ def create_ui():
             with gr.Column(variant='compact', elem_id="txt2img_settings"):
 
                 with gr.Row():
-                    width = gr.Slider(minimum=64, maximum=4096, step=8, label="Width", value=512, elem_id="txt2img_width")
-                    height = gr.Slider(minimum=64, maximum=4096, step=8, label="Height", value=512, elem_id="txt2img_height")
-                    res_switch_btn = ToolButton(value=ui_symbols.switch, elem_id="txt2img_res_switch_btn", label="Switch dims")
-                    res_switch_btn.click(lambda w, h: (h, w), inputs=[width, height], outputs=[width, height], show_progress=False)
+                    width, height = ui_sections.create_resolution_inputs('txt2img')
 
                 with gr.Group(elem_classes="settings-accordion"):
 

modules/upscaler.py

@@ -218,7 +218,11 @@ def compile_upscaler(model):
         if "Upscaler" in shared.opts.cuda_compile and shared.opts.cuda_compile_backend != 'none':
             import torch._dynamo # pylint: disable=unused-import,redefined-outer-name
             torch._dynamo.reset() # pylint: disable=protected-access
-            shared.log.debug(f"Upscaler compile available backends: {torch._dynamo.list_backends()}") # pylint: disable=protected-access
+            if shared.opts.cuda_compile_backend not in torch._dynamo.list_backends(): # pylint: disable=protected-access
+                shared.log.warning(f"Upscaler compile not available: backend={shared.opts.cuda_compile_backend} available={torch._dynamo.list_backends()}") # pylint: disable=protected-access
+                return model
+            else:
+                shared.log.info(f"Upscaler compile: backend={shared.opts.cuda_compile_backend} available={torch._dynamo.list_backends()}") # pylint: disable=protected-access
 
             if shared.opts.cuda_compile_backend == "openvino_fx":
                 from modules.intel.openvino import openvino_fx # pylint: disable=unused-import
@@ -241,11 +245,8 @@ def compile_upscaler(model):
                 shared.log.error(f"Torch inductor config error: {e}")
 
             t0 = time.time()
-
             model = torch.compile(model, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init
-
             setup_logging() # compile messes with logging so reset is needed
-
             t1 = time.time()
             shared.log.info(f"Upscaler compile: time={t1-t0:.2f}")
     except Exception as e:

modules/vqa.py

@@ -8,7 +8,7 @@ processor = None
 model = None
 loaded: str = None
 MODELS = {
-    "None": None,
+    "Moondream 2": "vikhyatk/moondream2", # 3.7GB
     "GIT TextCaps Base": "microsoft/git-base-textcaps", # 0.7GB
     "GIT VQA Base": "microsoft/git-base-vqav2", # 0.7GB
     "GIT VQA Large": "microsoft/git-large-vqav2", # 1.6GB
@@ -40,7 +40,6 @@ def git(question: str, image: Image.Image, repo: str = None):
         generated_ids = model.generate(**git_dict)
     response = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
 
-    model.to(devices.cpu)
     shared.log.debug(f'VQA: response={response}')
     return response
 
@@ -80,7 +79,6 @@ def vilt(question: str, image: Image.Image, repo: str = None):
     idx = logits.argmax(-1).item()
     response = model.config.id2label[idx]
 
-    model.to(devices.cpu)
     shared.log.debug(f'VQA: response={response}')
     return response
 
@@ -102,25 +100,50 @@ def pix(question: str, image: Image.Image, repo: str = None):
         outputs = model.generate(**inputs)
     response = processor.decode(outputs[0], skip_special_tokens=True)
 
-    model.to(devices.cpu)
     shared.log.debug(f'VQA: response={response}')
     return response
 
 
+def moondream(question: str, image: Image.Image, repo: str = None):
+    global processor, model, loaded # pylint: disable=global-statement
+    if model is None or loaded != repo:
+        model = transformers.AutoModelForCausalLM.from_pretrained(repo, trust_remote_code=True) # revision = "2024-03-05"
+        processor = transformers.AutoTokenizer.from_pretrained(repo) # revision = "2024-03-05"
+        loaded = repo
+        model.eval()
+    model.to(devices.device, devices.dtype)
+    shared.log.debug(f'VQA: class={model.__class__.__name__} processor={processor.__class__} model={repo}')
+
+    if len(question) < 2:
+        question = "Describe the image."
+    encoded = model.encode_image(image)
+    with devices.inference_context():
+        response = model.answer_question(encoded, question, processor)
+
+    shared.log.debug(f'VQA: response="{response}"')
+    return response
+
+
 def interrogate(vqa_question, vqa_image, vqa_model):
     vqa_model = MODELS.get(vqa_model, None)
-    shared.log.debug(f'VQA: model="{vqa_model}" question={vqa_question} image={vqa_image}')
+    shared.log.debug(f'VQA: model="{vqa_model}" question="{vqa_question}" image={vqa_image}')
     if vqa_image is None:
-        return 'no image provided'
+        answer = 'no image provided'
     if vqa_model is None:
-        return 'no model selected'
+        answer = 'no model selected'
     if 'git' in vqa_model.lower():
-        return git(vqa_question, vqa_image, vqa_model)
+        answer = git(vqa_question, vqa_image, vqa_model)
     if 'vilt' in vqa_model.lower():
-        return vilt(vqa_question, vqa_image, vqa_model)
+        answer = vilt(vqa_question, vqa_image, vqa_model)
     if 'blip' in vqa_model.lower():
-        return blip(vqa_question, vqa_image, vqa_model)
+        answer = blip(vqa_question, vqa_image, vqa_model)
     if 'pix' in vqa_model.lower():
-        return pix(vqa_question, vqa_image, vqa_model)
+        answer = pix(vqa_question, vqa_image, vqa_model)
+    if 'moondream2' in vqa_model.lower():
+        answer = moondream(vqa_question, vqa_image, vqa_model)
     else:
-        return 'unknown model'
+        answer = 'unknown model'
+    if model is not None:
+        model.to(devices.cpu)
+    devices.torch_gc()
+    return answer

modules/zluda.py

@@ -1,9 +1,16 @@
 import platform
 import torch
+from torch._prims_common import DeviceLikeType
 from modules import shared, devices
 
 
-def test(device: torch.device):
+def is_zluda(device: DeviceLikeType):
+    device = torch.device(device)
+    return torch.cuda.get_device_name(device).endswith("[ZLUDA]")
+
+
+def test(device: DeviceLikeType):
+    device = torch.device(device)
     try:
         ten1 = torch.randn((2, 4,), device=device)
         ten2 = torch.randn((4, 8,), device=device)
@@ -15,7 +22,7 @@ def test(device: torch.device):
 
 def initialize_zluda():
     device = devices.get_optimal_device()
-    if platform.system() == "Windows" and devices.cuda_ok and torch.cuda.get_device_name(device).endswith("[ZLUDA]"):
+    if platform.system() == "Windows" and devices.cuda_ok and is_zluda(device):
         torch.backends.cudnn.enabled = False
         torch.backends.cuda.enable_flash_sdp(False)
         torch.backends.cuda.enable_math_sdp(True)

pyproject.toml

@@ -14,7 +14,6 @@ exclude = [
   "modules/control/proc/leres/",
   "modules/control/units/*_model.py",
   "modules/control/units/*_pipe.py",
-  "modules/pipelines/*.py",
   "modules/xadapter/*.py",
   "modules/tcd/*.py",
 ]

scripts/stablevideodiffusion.py

@@ -62,9 +62,7 @@ class Script(scripts.Script):
             p.do_not_save_grid = True
             if c == 'StableVideoDiffusionPipeline':
                 p.sampler_name = 'Default' # svd does not support non-default sampler
-                p.task_args['output_type'] = 'np'
-            else:
-                p.task_args['output_type'] = 'pil'
+            p.task_args['output_type'] = 'pil'
             p.task_args['generator'] = torch.manual_seed(p.seed) # svd does not support gpu based generator
             p.task_args['width'] = p.width
             p.task_args['height'] = p.height