Merge pull request #3503 from vladmandic/dev

merge dev to master
2026-09-20 01:31:13 +02:00 · 2024-10-23 08:50:03 -04:00
parent 5c684cb076 6d4f2dfd7e
commit d1083e56a3
214 changed files with 16096 additions and 4245 deletions
@@ -123,6 +123,7 @@
    "venv",
    "panzoom.js",
    "split.js",
-    "exifr.js"
+    "exifr.js",
+    "iframeResizer.min.js"
  ]
 }
@@ -7,9 +7,13 @@ fail-on=
 fail-under=10
 ignore=CVS
 ignore-paths=/usr/lib/.*$,
+             modules/apg,
             modules/control/proc,
             modules/control/units,
+             modules/ctrlx,
+             modules/dcsolver,
             modules/dml,
+             modules/ggml,
             modules/hidiffusion,
             modules/hijack,
             modules/intel/ipex,
@@ -18,14 +22,16 @@ ignore-paths=/usr/lib/.*$,
             modules/ldsr,
             modules/onnx_impl,
             modules/pag,
+             modules/prompt_parser_xhinker.py,
             modules/rife,
             modules/taesd,
             modules/todo,
             modules/unipc,
+             modules/vdm,
             modules/xadapter,
-             modules/dcsolver,
+             modules/meissonic,
+             modules/omnigen,
             repositories,
-             modules/prompt_parser_xhinker.py,
             extensions-builtin/sd-webui-agent-scheduler,
             extensions-builtin/sd-extension-chainner/nodes,
             extensions-builtin/sdnext-modernui/node_modules,
@@ -163,12 +169,14 @@ disable=bad-inline-option,
        too-many-locals,
        too-many-nested-blocks,
        too-many-statements,
+        too-many-positional-arguments,
        unidiomatic-typecheck,
        unnecessary-dict-index-lookup,
        unnecessary-dunder-call,
        unnecessary-lambda,
        use-dict-literal,
        use-symbolic-message-instead,
+        unknown-option-value,
        useless-suppression,
        wrong-import-position,
 enable=c-extension-no-member
@@ -3,24 +3,29 @@ exclude = [
    ".git",
    ".ruff_cache",
    ".vscode",
+    "modules/apg",
+    "modules/control/proc",
+    "modules/control/units",
+    "modules/dcsolver",
+    "modules/ggml",
    "modules/hidiffusion",
    "modules/hijack",
+    "modules/intel/ipex",
+    "modules/intel/openvino",
    "modules/k-diffusion",
    "modules/ldsr",
    "modules/pag",
+    "modules/postprocess/aurasr_arch.py",
+    "modules/prompt_parser_xhinker.py",
    "modules/rife",
+    "modules/segmoe",
    "modules/taesd",
    "modules/todo",
    "modules/unipc",
+    "modules/vdm",
    "modules/xadapter",
-    "modules/dcsolver",
-    "modules/intel/openvino",
-    "modules/intel/ipex",
-    "modules/segmoe",
-    "modules/control/proc",
-    "modules/control/units",
-    "modules/prompt_parser_xhinker.py",
-    "modules/postprocess/aurasr_arch.py",
+    "modules/meissonic",
+    "modules/omnigen",
    "repositories",
    "extensions-builtin/sd-extension-chainner/nodes",
    "extensions-builtin/sd-webui-agent-scheduler",
@@ -65,6 +70,7 @@ ignore = [
  "E731",   # Do not assign a `lambda` expression, use a `def`
  "E741",   # Ambiguous variable name
  "F401",   # Imported by unused
+  "NPY002", # replace legacy random
  "RUF005", # Consider iterable unpacking
  "RUF010", # Use explicit conversion flag
  "RUF012", # Mutable class attributes
@@ -1,5 +1,315 @@
 # Change Log for SD.Next

+## Update for 2024-10-23
+
+### Highlights for 2024-10-23
+
+A month later and with nearly 300 commits, here is the latest [SD.Next](https://github.com/vladmandic/automatic) update!  
+
+#### Workflow highlights
+
+- **Reprocess**: New workflow options that allow you to generate at lower quality and then  
+  reprocess at higher quality for select images only or generate without hires/refine and then reprocess with hires/refine  
+  and you can pick any previous latent from auto-captured history!  
+- **Detailer** Fully built-in detailer workflow with support for all standard models  
+- Built-in **model analyzer**  
+  See all details of your currently loaded model, including components, parameter count, layer count, etc.  
+- **Extract LoRA**: load any LoRA(s) and play with generate as usual  
+  and once you like the results simply extract combined LoRA for future use!  
+
+#### New models
+
+- New fine-tuned [CLiP-ViT-L]((https://huggingface.co/zer0int/CLIP-GmP-ViT-L-14)) 1st stage **text-encoders** used by most models (SD15/SDXL/SD3/Flux/etc.) brings additional details to your images  
+- New models:  
+  [Stable Diffusion 3.5 Large](https://huggingface.co/stabilityai/stable-diffusion-3.5-large)  
+  [OmniGen](https://arxiv.org/pdf/2409.11340)  
+  [CogView 3 Plus](https://huggingface.co/THUDM/CogView3-Plus-3B)  
+  [Meissonic](https://github.com/viiika/Meissonic)  
+- Additional integration:  
+  [Ctrl+X](https://github.com/genforce/ctrl-x) which allows for control of **structure and appearance** without the need for extra models,  
+  [APG: Adaptive Projected Guidance](https://arxiv.org/pdf/2410.02416) for optimal **guidance** control,  
+  [LinFusion](https://github.com/Huage001/LinFusion) for on-the-fly **distillation** of any sd15/sdxl model  
+
+#### What else?
+
+- Tons of work on **dynamic quantization** that can be applied *on-the-fly* during model load to any model type (*you do not need to use pre-quantized models*)  
+  Supported quantization engines include `BitsAndBytes`, `TorchAO`, `Optimum.quanto`, `NNCF` compression, and more...  
+- Auto-detection of best available **device/dtype** settings for your platform and GPU reduces neeed for manual configuration  
+  *Note*: This is a breaking change to default settings and its recommended to check your preferred settings after upgrade  
+- Full rewrite of **sampler options**, not far more streamlined with tons of new options to tweak scheduler behavior  
+- Improved **LoRA** detection and handling for all supported models  
+- Several of [Flux.1](https://huggingface.co/black-forest-labs/FLUX.1-dev) optimizations and new quantization types  
+
+Oh, and we've compiled a full table with list of top-30 (*how many have you tried?*) popular text-to-image generative models,  
+their respective parameters and architecture overview: [Models Overview](https://github.com/vladmandic/automatic/wiki/Models)  
+
+And there are also other goodies like multiple *XYZ grid* improvements, additional *Flux ControlNets*, additional *Interrogate models*, better *LoRA tags* support, and more...  
+[README](https://github.com/vladmandic/automatic/blob/master/README.md) | [CHANGELOG](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867)
+
+
+### Details for 2024-10-23
+
+- **reprocess**
+  - new top-level button: reprocess latent from your history of generated image(s)  
+  - generate using full-quality:off and then reprocess using *full quality decode*  
+  - generate without hires/refine and then *reprocess with hires/refine*  
+    *note*: you can change hires/refine settings and run-reprocess again!  
+  - reprocess using *detailer*  
+
+- **history**
+  - by default, **reprocess** will pick last latent, but you can select any latent from history!  
+  - history is under *networks -> history*  
+    each history item includes info on operations that were used, timestamp and metadata  
+  - any latent operation during workflow automatically adds one or more items to history  
+    e.g. generate base + upscale + hires + detailer  
+  - history size: *settings -> execution -> latent history size*  
+    memory usage is ~130kb of ram for 1mp image  
+  - *note* list of latents in history is not auto-refreshed, use refresh button  
+
+- **model analyzer**  
+  - see all details of your currently loaded model, including components, parameter count, layer count, etc.  
+  - in models -> current -> analyze  
+
+- **text encoder**:  
+  - allow loading different custom text encoders: *clip-vit-l, clip-vit-g, t5*  
+    will automatically find appropriate encoder in the loaded model and replace it with loaded text encoder  
+    download text encoders into folder set in settings -> system paths -> text encoders  
+    default `models/Text-encoder` folder is used if no custom path is set  
+    finetuned *clip-vit-l* models: [Detailed, Smooth](https://huggingface.co/zer0int/CLIP-GmP-ViT-L-14), [LongCLIP](https://huggingface.co/zer0int/LongCLIP-GmP-ViT-L-14)  
+    reference *clip-vit-l* and *clip-vit-g* models: [OpenCLIP-Laion2b](https://huggingface.co/collections/laion/openclip-laion-2b-64fcade42d20ced4e9389b30)  
+    *note* sd/sdxl contain heavily distilled versions of reference models, so switching to reference model produces vastly different results  
+  - xyz grid support for text encoder  
+  - full prompt parser now correctly works with different prompts in batch  
+
+- **detailer**:  
+  - replaced *face-hires* with *detailer* which can run any number of standard detailing models  
+  - includes *face/hand/person/eyes* predefined detailer models plus support for manually downloaded models  
+    set path in *settings -> system paths -> yolo*  
+  - select one or more models in detailer menu and thats it!  
+  - to avoid duplication of ui elements, detailer will use following values from **refiner**:  
+    *sampler, steps, prompts*  
+  - when using multiple detailers and prompt is *multi-line*, each line is applied to corresponding detailer  
+  - adjustable settings:  
+    *strength, max detected objects, edge padding, edge blur, min detection confidence, max detection overlap, min and max size of detected object*  
+  - image metadata includes info on used detailer models  
+  - *note* detailer defaults are not save in ui settings, they are saved in server settings  
+    to apply your defaults, set ui values and apply via *system -> settings -> apply settings*  
+  - if using models trained on multiple classes, you can specify which classes you want to detail  
+    e.g. original yolo detection model is trained on coco dataset with 80 predefined classes  
+    if you leave field blank, it will use any class found in the model  
+    you can see classes defined in the model while model itself is loaded for the first time  
+
+- **extract lora**: extract combined lora from current memory state, thanks @AI-Casanova  
+  load any LoRA(s) and play with generate as usual and once you like the results simply extract combined LoRA for future use!  
+  in *models -> extract lora*  
+
+- **sampler options**: full rewrite  
+
+  *sampler notes*:  
+  - pick a sampler and then pick values, all values have "default" as a choice to make it simpler  
+  - a lot of options are new, some are old but moved around  
+    e.g. karras checkbox is replaced with a choice of different sigma methods  
+  - not every combination of settings is valid  
+  - some settings are specific to model types  
+    e.g. sd15/sdxl typically use epsilon prediction  
+  - quite a few well-known schedulers are just variations of settings, for example:  
+    - *sampler sgm* is sampler with trailing spacing and sample prediction type  
+    - *dpm 2m* or *3m* are *dpm 1s* with orders of 2 or 3  
+    - *dpm 2m sde* is *dpm 2m* with *sde* as solver  
+    - *sampler simple* is sampler with trailing spacing and linear beta schedule
+  - xyz grid support for sampler options  
+  - metadata updates for sampler options  
+  - modernui updates for sampler options  
+  - *note* sampler options defaults are not save in ui settings, they are saved in server settings  
+    to apply your defaults, set ui values and apply via *system -> settings -> apply settings*  
+
+  *sampler options*:  
+  - sigma method: *karas, beta, exponential*  
+  - timesteps spacing: *linspace, leading, trailing*  
+  - beta schedule: *linear, scaled, cosine*  
+  - prediction type: *epsilon, sample, v-prediction*  
+  - timesteps presents: *none, ays-sd15, ays-sdxl*  
+  - timesteps override: <custom>  
+  - sampler order: *0=default, 1-5*  
+  - options: *dynamic, low order, rescale*  
+
+- [Ctrl+X](https://github.com/genforce/ctrl-x):
+  - control **structure** (*similar to controlnet*) and **appearance** (*similar to ipadapter*)  
+    without the need for extra models, all via code feed-forwards!
+  - can run in structure-only or appearance-only or both modes
+  - when providing structure and appearance input images, its best to provide a short prompts describing them  
+  - structure image can be *almost anything*: *actual photo, openpose-style stick man, 3d render, sketch, depth-map, etc.*  
+    just describe what it is in a structure prompt so it can be de-structured and correctly applied  
+  - supports sdxl in both txt2img and img2img, simply select from scripts
+
+- [APG: Adaptive Projected Guidance](https://arxiv.org/pdf/2410.02416)
+  - latest algo to provide better guidance for image generation, can be used instead of existing guidance rescale and/or PAG  
+  - in addtion to stronger guidance and reduction of burn at high guidance values, it can also increase image details  
+  - compatible with *sd15/sdxl/sc*  
+  - select in scripts -> apg  
+  - for low    cfg scale, use positive momentum: e.g. cfg=2 => momentum=0.6
+  - for normal cfg scale, use negative momentum: e.g. cfg=6 => momentum=-0.3
+  - for high   cfg scale, use neutral  momentum: e.g. cfg=10 => momentum=0.0
+
+- [LinFusion](https://github.com/Huage001/LinFusion)  
+  - apply liner distillation to during load to any sd15/sdxl model  
+  - can reduce vram use for high resolutions and increase performance
+  - *note*: use lower cfg scales as typical for distilled models  
+
+- [Flux](https://huggingface.co/black-forest-labs/FLUX.1-dev)  
+  - see [wiki](https://github.com/vladmandic/automatic/wiki/FLUX#quantization) for details on `gguf`  
+  - support for `gguf` binary format for loading unet/transformer component  
+  - support for `gguf` binary format for loading t5/text-encoder component: requires transformers pr  
+  - additional controlnets: [JasperAI](https://huggingface.co/collections/jasperai/flux1-dev-controlnets-66f27f9459d760dcafa32e08) **Depth**, **Upscaler**, **Surface**, thanks @EnragedAntelope  
+  - additional controlnets: [XLabs-AI](https://huggingface.co/XLabs-AI/flux-controlnet-hed-diffusers) **Canny**, **Depth**, **HED**  
+  - mark specific unet as unavailable if load failed  
+  - fix diffusers local model name parsing  
+  - full prompt parser will auto-select `xhinker` for flux models  
+  - controlnet support for img2img and inpaint (in addition to previous txt2img controlnet)  
+  - allow separate vae load  
+  - support for both kohya and onetrainer loras in native load mode for fp16/nf4/fp4, thanks @AI-Casanova  
+  - support for differential diffusion  
+  - added native load mode for qint8/qint4 models
+  - avoid unet load if unchanged  
+
+- [OmniGen](https://arxiv.org/pdf/2409.11340)  
+  - Radical new model with pure LLM architecture based on Phi-3  
+  - Select from *networks -> models -> reference*  
+  - Can be used for text-to-image and image-to-image  
+  - Image-to-image is *very* different, you need to specify in prompt what do you want to do  
+    and add `|image|` placeholder where input image is used!  
+    examples: `in |image| remove glasses from face`, `using depth map from |image|, create new image of a cute robot`  
+  - Params used: prompt, steps, guidance scale for prompt guidance, refine guidance scale for image guidance  
+    Recommended: guidance=3.0, refine-guidance=1.6  
+
+- [Stable Diffusion 3.5 Large](https://huggingface.co/stabilityai/stable-diffusion-3.5-large)  
+  - New/improved variant of Stable Diffusion 3  
+  - Select from *networks -> models -> reference*  
+  - Available in standard and turbo variations  
+  - *Note*: Access to to both variations of SD3.5 model is gated, you must accept the conditions and use HF login  
+
+- [CogView 3 Plus](https://huggingface.co/THUDM/CogView3-Plus-3B)
+  - Select from *networks -> models -> reference*  
+  - resolution width and height can be from 512px to 2048px and must be divisible by 32  
+  - precision: bf16 or fp32  
+    fp16 is not supported due to internal model overflows  
+
+- [Meissonic](https://github.com/viiika/Meissonic)  
+  - Select from *networks -> models -> reference*  
+  - Experimental as upstream implemenation code is unstable
+  - Must set scheduler:default, generator:unset
+
+- [SageAttention](https://github.com/thu-ml/SageAttention)  
+  - new 8-bit attention implementation on top of SDP that can provide acceleration for some models, thanks @Disty0  
+  - enable in *settings -> compute settings -> sdp options -> sage attention*
+  - compatible with DiT-based models: e.g. *Flux.1, AuraFlow, CogVideoX*  
+  - not compatible with UNet-based models, e.g. *SD15, SDXL*  
+
+- **gpu**
+  - previously `cuda_dtype` in settings defaulted to `fp16` if available  
+  - now `cuda_type` defaults to **Auto** which executes `bf16` and `fp16` tests on startup and selects best available dtype  
+    if you have specific requirements, you can still set to fp32/fp16/bf16 as desired  
+    if you have gpu that incorrectly identifies bf16 or fp16 availablity, let us know so we can improve the auto-detection  
+  - support for torch **expandable segments**  
+    enable in *settings -> compute -> torch expandable segments*  
+    can provide significant memory savings for some models  
+    not enabled by default as its only supported on latest versions of torch and some gpus  
+
+- **xyz grid** full refactor  
+  - multi-mode: *selectable-script* and *alwayson-script*  
+  - allow usage combined with other scripts  
+  - allow **unet** selection  
+  - allow passing **model args** directly:  
+    allowed params will be checked against models call signature  
+    example: `width=768; height=512, width=512; height=768`  
+  - allow passing **processing args** directly:  
+    params are set directly on main processing object and can be known or new params  
+    example: `steps=10, steps=20; test=unknown`  
+  - enable working with different resolutions  
+    now you can adjust width/height in the grid just as any other param  
+  - renamed options to include section name and adjusted cost of each option  
+  - added additional metadata  
+
+- **interrogate**  
+  - add additional blip models: *blip-base, blip-large, blip-t5-xl, blip-t5-xxl, opt-2.7b, opt-6.7b*  
+  - change default params for better memory utilization  
+  - lock commits for miaoshouAI-promptgen  
+  - add optional advanced params  
+  - update logging  
+
+- **lora** auto-apply tags to prompt  
+  - controlled via *settings -> networks -> lora_apply_tags*  
+    *0:disable, -1:all-tags, n:top-n-tags*  
+  - uses tags from both model embedded data and civitai downloaded data  
+  - if lora contains no tags, lora name itself will be used as a tag  
+  - if prompt contains `_tags_` it will be used as placeholder for replacement, otherwise tags will be appended  
+  - used tags are also logged and registered in image metadata  
+  - loras are no longer filtered per detected type vs loaded model type as its unreliable  
+  - loras display in networks now shows possible version in top-left corner  
+  - correct using of `extra_networks_default_multiplier` if not scale is specified  
+  - improve lora base model detection  
+  - improve lora error handling and logging  
+  - setting `lora_load_gpu` to load LoRA directly to GPU  
+    *default*: true unless lovwram  
+
+- **quantization**  
+  - new top level settings group as we have quite a few quantization options now!  
+    configure in *settings -> quantization*  
+  - in addition to existing `optimum.quanto` and `nncf`, we now have `bitsandbytes` and `torchao`  
+  - **bitsandbytes**: fp8, fp4, nf4  
+    - quantization can be applied on-the-fly during model load  
+    - currently supports `transformers` and `t5` in **sd3** and **flux**  
+  - **torchao**: int8, int4, fp8, fp4, fpx  
+    - configure in settings -> quantization  
+    - can be applied to any model on-the-fly during load  
+
+- **huggingface**:  
+  - force logout/login on token change  
+  - unified handling of cache folder: set via `HF_HUB` or `HF_HUB_CACHE` or via settings -> system paths  
+
+- **cogvideox**:  
+  - add support for *image2video* (in addition to previous *text2video* and *video2video*)  
+  - *note*: *image2video* requires separate 5b model variant  
+
+- **torch**  
+  - due to numerous issues with torch 2.5.0 which was just released as stable, we are sticking with 2.4.1 for now  
+
+- **backend=original** is now marked as in maintenance-only mode  
+- **python 3.12** improved compatibility, automatically handle `setuptools`  
+- **control**
+  - persist/reapply units current state on server restart  
+  - better handle size before/after metadata  
+- **video** add option `gradio_skip_video` to avoid gradio issues with displaying generated videos  
+- add support for manually downloaded diffusers models from huggingface  
+- **ui**  
+  - move checkboxes `full quality, tiling, hidiffusion` to advanced section  
+  - hide token counter until tokens are known  
+  - minor ui optimizations  
+  - fix update infotext on image select  
+  - fix imageviewer exif parser  
+  - selectable info view in image viewer, thanks @ZeldaMaster501  
+  - setting to enable browser autolaunch, thanks @brknsoul  
+- **free-u** check if device/dtype are fft compatible and cast as necessary  
+- **rocm**
+  - additional gpu detection and auto-config code, thanks @lshqqytiger  
+  - experimental triton backend for flash attention, thanks @lshqqytiger  
+  - update to rocm 6.2, thanks @Disty0
+- **directml**  
+  - update `torch` to 2.4.1, thanks @lshqqytiger  
+- **extensions**  
+  - add mechanism to lock-down extension to specific working commit  
+  - added `sd-webui-controlnet` and `adetailer` last-known working commits  
+- **upscaling**  
+  - interruptible operations
+- **refactor**  
+  - general lora apply/unapply process  
+  - modularize main process loop  
+  - massive log cleanup  
+  - full lint pass  
+  - improve inference mode handling  
+  - unify quant lib loading  
+
+
 ## Update for 2024-09-13

 ### Highlights for 2024-09-13
@@ -105,6 +415,9 @@ Examples:
 - **prompt enhance**: improve quality and/or verbosity of your prompts  
  simply select in *scripts -> prompt enhance*
  uses [gokaygokay/Flux-Prompt-Enhance](https://huggingface.co/gokaygokay/Flux-Prompt-Enhance) model  
+- **decode**
+  - auto-set upcast if first decode fails  
+  - restore dtype on upcast  
 - **taesd** configurable number of layers  
  can be used to speed-up taesd decoding by reducing number of ops  
  e.g. if generating 1024px image, reducing layers by 1 will result in preview being 512px  
@@ -31,7 +31,7 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG
 - Multiple UIs!  
  ▹ **Standard | Modern**  
 - Multiple diffusion models!  
-  ▹ **Stable Diffusion 1.5/2.1/XL/3.0 | LCM | Lightning | Segmind | Kandinsky | Pixart-α | Pixart-Σ | Stable Cascade | FLUX.1 | AuraFlow | Würstchen | Lumina | Kolors | aMUSEd | DeepFloyd IF | UniDiffusion | SD-Distilled | BLiP Diffusion | KOALA | SDXS | Hyper-SD | HunyuanDiT | etc.**
+  ▹ **Stable Diffusion 1.5/2.1/XL/3.0/3.5 | LCM | Lightning | Segmind | Kandinsky | Pixart-α | Pixart-Σ | Stable Cascade | FLUX.1 | AuraFlow | Würstchen | Alpha Lumina | Kwai Kolors | aMUSEd | DeepFloyd IF | UniDiffusion | SD-Distilled | BLiP Diffusion | KOALA | SDXS | Hyper-SD | HunyuanDiT | CogView | OmniGen | Meissonic | etc.**
 - Built-in Control for Text, Image, Batch and video processing!  
  ▹ **ControlNet | ControlNet XS | Control LLLite | T2I Adapters | IP Adapters**  
 - Multiplatform!  
@@ -68,27 +68,31 @@ Additional models will be added as they become available and there is public int
 - [RunwayML Stable Diffusion](https://github.com/Stability-AI/stablediffusion/) 1.x and 2.x *(all variants)*
 - [StabilityAI Stable Diffusion XL](https://github.com/Stability-AI/generative-models)
 - [StabilityAI Stable Diffusion 3 Medium](https://stability.ai/news/stable-diffusion-3-medium)
+- [Stable Diffusion 3.5 Large](https://huggingface.co/stabilityai/stable-diffusion-3.5-large)  
 - [StabilityAI Stable Video Diffusion](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid) Base, XT 1.0, XT 1.1
- [LCM: Latent Consistency Models](https://github.com/openai/consistency_models)
+- [StabilityAI Stable Cascade](https://github.com/Stability-AI/StableCascade) *Full* and *Lite*
 - [Black Forest Labs FLUX.1](https://blackforestlabs.ai/announcing-black-forest-labs/) Dev, Schnell  
 - [AuraFlow](https://huggingface.co/fal/AuraFlow)
 - [AlphaVLLM Lumina-Next-SFT](https://huggingface.co/Alpha-VLLM/Lumina-Next-SFT-diffusers)  
+- [Playground AI](https://huggingface.co/playgroundai/playground-v2-256px-base) *v1, v2 256, v2 512, v2 1024 and latest v2.5*
+- [Tencent HunyuanDiT](https://github.com/Tencent/HunyuanDiT)
+- [OmniGen](https://arxiv.org/pdf/2409.11340)  
+- [Meissonic](https://github.com/viiika/Meissonic)  
 - [Kwai Kolors](https://huggingface.co/Kwai-Kolors/Kolors)  
- [Playground](https://huggingface.co/playgroundai/playground-v2-256px-base) *v1, v2 256, v2 512, v2 1024 and latest v2.5*
- [Stable Cascade](https://github.com/Stability-AI/StableCascade) *Full* and *Lite*
- [aMUSEd 256](https://huggingface.co/amused/amused-256) 256 and 512
+- [CogView 3+](https://huggingface.co/THUDM/CogView3-Plus-3B)
+- [LCM: Latent Consistency Models](https://github.com/openai/consistency_models)
+- [aMUSEd](https://huggingface.co/amused/amused-256) 256 and 512
 - [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega)
 - [Segmind SSD-1B](https://huggingface.co/segmind/SSD-1B)
 - [Segmind SegMoE](https://github.com/segmind/segmoe) *SD and SD-XL*
+- [Segmind SD Distilled](https://huggingface.co/blog/sd_distillation) *(all variants)*
 - [Kandinsky](https://github.com/ai-forever/Kandinsky-2) *2.1 and 2.2 and latest 3.0*
 - [PixArt-α XL 2](https://github.com/PixArt-alpha/PixArt-alpha) *Medium and Large*
 - [PixArt-Σ](https://github.com/PixArt-alpha/PixArt-sigma)
 - [Warp Wuerstchen](https://huggingface.co/blog/wuertschen)
- [Tencent HunyuanDiT](https://github.com/Tencent/HunyuanDiT)
 - [Tsinghua UniDiffusion](https://github.com/thu-ml/unidiffuser)
 - [DeepFloyd IF](https://github.com/deep-floyd/IF) *Medium and Large*
 - [ModelScope T2V](https://huggingface.co/damo-vilab/text-to-video-ms-1.7b)
- [Segmind SD Distilled](https://huggingface.co/blog/sd_distillation) *(all variants)*
 - [BLIP-Diffusion](https://dxli94.github.io/BLIP-Diffusion-website/)
 - [KOALA 700M](https://github.com/youngwanLEE/sdxl-koala)
 - [VGen](https://huggingface.co/ali-vilab/i2vgen-xl)
@@ -49,7 +49,7 @@ def generate(args): # pylint: disable=redefined-outer-name
    options['width'] = int(args.width)
    options['height'] = int(args.height)
    if args.faces:
-        options['restore_faces'] = args.faces
+        options['detailer'] = args.detailer
        options['denoising_strength'] = 0.5
        options['hr_sampler_name'] = args.sampler
    data = post('/sdapi/v1/txt2img', options)
@@ -75,7 +75,7 @@ if __name__ == "__main__":
    parser.add_argument('--height', required=False, default=512, help='image height')
    parser.add_argument('--steps', required=False, default=20, help='number of steps')
    parser.add_argument('--seed', required=False, default=-1, help='initial seed')
-    parser.add_argument('--faces', action='store_true', help='restore faces')
+    parser.add_argument('--detailer', action='store_true', help='run detailer')
    parser.add_argument('--sampler', required=False, default='Euler a', help='sampler name')
    parser.add_argument('--output', required=False, default=None, help='output image file')
    parser.add_argument('--model', required=False, help='model name')
@@ -46,7 +46,7 @@ options = Map({
    },
    # generate params
    'generate': {
-        'restore_faces': True,
+        'detailer': True,
        'prompt': '',
        'negative_prompt': 'foggy, blurry, blurred, duplicate, ugly, mutilated, mutation, mutated, out of frame, bad anatomy, disfigured, deformed, censored, low res, low resolution, watermark, text, poorly drawn face, poorly drawn hands, signature',
        'steps': 20,
@@ -8,7 +8,7 @@
  },
  "generate":
  {
-      "restore_faces": true,
+      "detailer": true,
      "prompt": "dynamic",
      "negative_prompt": "foggy, blurry, blurred, duplicate, ugly, mutilated, mutation, mutated, out of frame, bad anatomy, disfigured, deformed, censored, low res, watermark, text, poorly drawn face, signature",
      "steps": 30,
@@ -230,7 +230,7 @@ def args(): # parse cmd arguments
    parser.add_argument('--style', type = str, default = 'random', required = False, help = 'image style, used to guide dynamic prompt when prompt is not provided')
    parser.add_argument('--suffix', type = str, default = 'random', required = False, help = 'style suffix, used to guide dynamic prompt when prompt is not provided')
    parser.add_argument('--place', type = str, default = 'random', required = False, help = 'place locator, used to guide dynamic prompt when prompt is not provided')
-    parser.add_argument('--faces', default = False, action='store_true', help = 'restore faces during upscaling')
+    parser.add_argument('--detailer', default = False, action='store_true', help = 'run detailer')
    parser.add_argument('--steps', type = int, default = 0, required = False, help = 'number of steps')
    parser.add_argument('--batch', type = int, default = 0, required = False, help = 'batch size, limited by gpu vram')
    parser.add_argument('--n', type = int, default = 0, required = False, help = 'number of iterations')
@@ -299,7 +299,7 @@ def args(): # parse cmd arguments
        _dynamic = prompt(params)

    sd.paths.root = params.path if params.path != '' else sd.paths.root
-    sd.generate.restore_faces = params.faces if params.faces is not None else sd.generate.restore_faces
+    sd.generate.detailer = params.detailer if params.detailer is not None else sd.generate.detailer
    sd.generate.seed = params.seed if params.seed > 0 else sd.generate.seed
    sd.generate.sampler_name = params.sampler if params.sampler != 'random' else sd.generate.sampler_name
    sd.generate.batch_size = params.batch if params.batch > 0 else sd.generate.batch_size
@@ -309,7 +309,7 @@ def args(): # parse cmd arguments
    sd.generate.height = params.height if params.height > 0 else sd.generate.height
    sd.generate.steps = params.steps if params.steps > 0 else sd.generate.steps
    sd.upscale.upscaling_resize = params.upscale if params.upscale > 0 else sd.upscale.upscaling_resize
-    sd.upscale.codeformer_visibility = 1 if params.faces else sd.upscale.codeformer_visibility
+    sd.upscale.codeformer_visibility = 1 if params.detailer else sd.upscale.codeformer_visibility
    sd.options.sd_vae = params.vae if params.vae != '' else sd.options.sd_vae
    sd.options.sd_model_checkpoint = params.model if params.model != '' else sd.options.sd_model_checkpoint
    sd.upscale.upscaler_1 = 'SwinIR_4x' if params.upscale > 1 else sd.upscale.upscaler_1
@@ -120,3 +120,5 @@ if __name__ == '__main__':
            for root, _dirs, files in os.walk(fn):
                for file in files:
                    read_exif(os.path.join(root, file))
+        else:
+            print('file not found: ', fn)
@@ -0,0 +1,89 @@
+import torch
+import diffusers
+
+
+class StateDictStats():
+    cls: str = None
+    device: torch.device = None
+    params: int = 0
+    weights: dict = {}
+    dtypes: dict = {}
+    config: dict = None
+
+    def __repr__(self):
+        return f'cls={self.cls} params={self.params} weights={self.weights} device={self.device} dtypes={self.dtypes} config={self.config is not None}'
+
+
+def set_module_tensor(
+    module: torch.nn.Module,
+    name: str,
+    value: torch.Tensor,
+    stats: StateDictStats,
+    device: torch.device = None,
+    dtype: torch.dtype = None,
+):
+    if "." in name:
+        splits = name.split(".")
+        for split in splits[:-1]:
+            module = getattr(module, split)
+        name = splits[-1]
+    old_value = getattr(module, name)
+    with torch.no_grad():
+        if value.dtype not in stats.dtypes:
+            stats.dtypes[value.dtype] = 0
+        stats.dtypes[value.dtype] += 1
+        if name in module._buffers: # pylint: disable=protected-access
+            module._buffers[name] = value.to(device=device, dtype=dtype, non_blocking=True) # pylint: disable=protected-access
+            if 'buffers' not in stats.weights:
+                stats.weights['buffers'] = 0
+            stats.weights['buffers'] += 1
+        elif value is not None:
+            param_cls = type(module._parameters[name]) # pylint: disable=protected-access
+            module._parameters[name] = param_cls(value, requires_grad=old_value.requires_grad).to(device, dtype=dtype, non_blocking=True) # pylint: disable=protected-access
+            if 'parameters' not in stats.weights:
+                stats.weights['parameters'] = 0
+            stats.weights['parameters'] += 1
+
+
+def load_unet(config_file: str, state_dict: dict, device: torch.device = None, dtype: torch.dtype = None):
+    # same can be done for other modules or even for entire model by loading model config and then walking through its modules
+    from accelerate import init_empty_weights
+    with init_empty_weights():
+        stats = StateDictStats()
+        stats.device = device
+        stats.config = diffusers.UNet2DConditionModel.load_config(config_file)
+        unet = diffusers.UNet2DConditionModel.from_config(stats.config)
+        stats.cls = unet.__class__.__name__
+        expected_state_dict_keys = list(unet.state_dict().keys())
+        stats.weights['expected'] = len(expected_state_dict_keys)
+    for param_name, param in state_dict.items():
+        if param_name not in expected_state_dict_keys:
+            if 'unknown' not in stats.weights:
+                stats.weights['unknown'] = 0
+            stats.weights['unknown'] += 1
+            continue
+        set_module_tensor(unet, name=param_name, value=param, device=device, dtype=dtype, stats=stats)
+        state_dict[param_name] = None # unload as we initialize the model so we dont consume double the memory
+    stats.params = sum(p.numel() for p in unet.parameters(recurse=True))
+    return unet, stats
+
+
+def load_safetensors(fn: str):
+    import safetensors.torch
+    state_dict = safetensors.torch.load_file(fn, device='cpu') # state dict should always be loaded to cpu
+    return state_dict
+
+
+if __name__ == "__main__":
+    # need pipe already present to load unet state_dict into or we could load unet first and then manually create pipe with params
+    pipe = diffusers.StableDiffusionXLPipeline.from_single_file('/mnt/models/stable-diffusion/sdxl/TempestV0.1-Artistic.safetensors', cache_dir='/mnt/models/huggingface')
+    # this could be kept in memory so we dont have to reload it
+    dct = load_safetensors('/mnt/models/UNET/dpo-sdxl-text2image.safetensors')
+    pipe.unet, s = load_unet(
+        config_file = 'configs/sdxl/unet/config.json', # can also point to online hf model with subfolder
+        state_dict = dct,
+        device = torch.device('cpu'), # can leave out to use default device
+        dtype = torch.bfloat16, # can leave out to use default dtype, especially for mixed precision modules
+    )
+    from rich import print as rprint
+    rprint(f'Stats: {s}')
@@ -7,22 +7,27 @@ from rich import print # pylint: disable=redefined-builtin

 def read_metadata(fn):
    res = {}
+    if not fn.lower().endswith(".safetensors"):
+        return
    with open(fn, mode="rb") as f:
-        metadata_len = f.read(8)
-        metadata_len = int.from_bytes(metadata_len, "little")
-        json_start = f.read(2)
-        if metadata_len <= 2 or json_start not in (b'{"', b"{'"):
-            print(f"Not a valid safetensors file: {fn}")
-        json_data = json_start + f.read(metadata_len-2)
-        json_obj = json.loads(json_data)
-        for k, v in json_obj.get("__metadata__", {}).items():
-            res[k] = v
-            if isinstance(v, str) and v[0:1] == '{':
-                try:
-                    res[k] = json.loads(v)
-                except Exception:
-                    pass
-    print(f"{fn}: {json.dumps(res, indent=4)}")
+        try:
+            metadata_len = f.read(8)
+            metadata_len = int.from_bytes(metadata_len, "little")
+            json_start = f.read(2)
+            if metadata_len <= 2 or json_start not in (b'{"', b"{'"):
+                print(f"Not a valid safetensors file: {fn}")
+            json_data = json_start + f.read(metadata_len-2)
+            json_obj = json.loads(json_data)
+            for k, v in json_obj.get("__metadata__", {}).items():
+                res[k] = v
+                if isinstance(v, str) and v[0:1] == '{':
+                    try:
+                        res[k] = json.loads(v)
+                    except Exception:
+                        pass
+            print(f"{fn}: {json.dumps(res, indent=4)}")
+        except Exception:
+            print(f"{fn}: cannot read metadata")


 def main():
@@ -5,31 +5,23 @@ import networks
 import lora_patches
 from modules import extra_networks, shared

+
 # from https://github.com/cheald/sd-webui-loractl/blob/master/loractl/lib/utils.py
 def get_stepwise(param, step, steps):
    def sorted_positions(raw_steps):
        steps = [[float(s.strip()) for s in re.split("[@~]", x)]
                 for x in re.split("[,;]", str(raw_steps))]
-        # If we just got a single number, just return it
-        if len(steps[0]) == 1:
+        if len(steps[0]) == 1: # If we just got a single number, just return it
            return steps[0][0]
-
-        # Add implicit 1s to any steps which don't have a weight
-        steps = [[s[0], s[1] if len(s) == 2 else 1] for s in steps]
-
-        # Sort by index
-        steps.sort(key=lambda k: k[1])
-
+        steps = [[s[0], s[1] if len(s) == 2 else 1] for s in steps] # Add implicit 1s to any steps which don't have a weight
+        steps.sort(key=lambda k: k[1]) # Sort by index
        steps = [list(v) for v in zip(*steps)]
        return steps

    def calculate_weight(m, step, max_steps, step_offset=2):
        if isinstance(m, list):
            if m[1][-1] <= 1.0:
-                if max_steps > 0:
-                    step = (step) / (max_steps - step_offset)
-                else:
-                    step = 1.0
+                step = (step) / (max_steps - step_offset) if max_steps > 0 else 1.0
            v = np.interp(step, m[1], m[0])
            return v
        else:
@@ -42,19 +34,53 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
    def __init__(self):
        super().__init__('lora')
        self.active = False
+        self.model = None
        self.errors = {}
        networks.originals = lora_patches.LoraPatches()

-        """mapping of network names to the number of errors the network had during operation"""
+    def prompt(self, p):
+        if shared.opts.lora_apply_tags == 0:
+            return
+        all_tags = []
+        for loaded in networks.loaded_networks:
+            page = [en for en in shared.extra_networks if en.name == 'lora'][0]
+            item = page.create_item(loaded.name)
+            tags = (item or {}).get("tags", {})
+            loaded.tags = list(tags)
+            if len(loaded.tags) == 0:
+                loaded.tags.append(loaded.name)
+            if shared.opts.lora_apply_tags > 0:
+                loaded.tags = loaded.tags[:shared.opts.lora_apply_tags]
+            all_tags.extend(loaded.tags)
+        if len(all_tags) > 0:
+            shared.log.debug(f"Load network: type=LoRA tags={all_tags} max={shared.opts.lora_apply_tags} apply")
+            all_tags = ', '.join(all_tags)
+            p.extra_generation_params["LoRA tags"] = all_tags
+            if '_tags_' in p.prompt:
+                p.prompt = p.prompt.replace('_tags_', all_tags)
+            else:
+                p.prompt = f"{p.prompt}, {all_tags}"
+            if p.all_prompts is not None:
+                for i in range(len(p.all_prompts)):
+                    if '_tags_' in p.all_prompts[i]:
+                        p.all_prompts[i] = p.all_prompts[i].replace('_tags_', all_tags)
+                    else:
+                        p.all_prompts[i] = f"{p.all_prompts[i]}, {all_tags}"

-    def activate(self, p, params_list, step=0):
-        t0 = time.time()
-        self.errors.clear()
-        if len(params_list) > 0:
-            self.active = True
-            networks.originals.apply() # apply patches
-            if networks.debug:
-                shared.log.debug("LoRA activate")
+    def infotext(self, p):
+        names = [i.name for i in networks.loaded_networks]
+        if len(names) > 0:
+            p.extra_generation_params["LoRA networks"] = ", ".join(names)
+        if shared.opts.lora_add_hashes_to_infotext:
+            network_hashes = []
+            for item in networks.loaded_networks:
+                if not item.network_on_disk.shorthash:
+                    continue
+                network_hashes.append(item.network_on_disk.shorthash)
+            if len(network_hashes) > 0:
+                p.extra_generation_params["LoRA hashes"] = ", ".join(network_hashes)
+
+    def parse(self, p, params_list, step=0):
        names = []
        te_multipliers = []
        unet_multipliers = []
@@ -62,7 +88,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
        for params in params_list:
            assert params.items
            names.append(params.positional[0])
-            te_multiplier = params.named.get("te", params.positional[1] if len(params.positional) > 1 else 1.0)
+            te_multiplier = params.named.get("te", params.positional[1] if len(params.positional) > 1 else shared.opts.extra_networks_default_multiplier)
            if isinstance(te_multiplier, str) and "@" in te_multiplier:
                te_multiplier = get_stepwise(te_multiplier, step, p.steps)
            else:
@@ -82,46 +108,44 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
            te_multipliers.append(te_multiplier)
            unet_multipliers.append(unet_multiplier)
            dyn_dims.append(dyn_dim)
-        t1 = time.time()
+        return names, te_multipliers, unet_multipliers, dyn_dims
+
+    def activate(self, p, params_list, step=0):
+        t0 = time.time()
+        self.errors.clear()
+        if self.active:
+            if self.model != shared.opts.sd_model_checkpoint: # reset if model changed
+                self.active = False
+        if len(params_list) > 0 and not self.active: # activate patches once
+            shared.log.debug(f'Activate network: type=LoRA model="{shared.opts.sd_model_checkpoint}"')
+            networks.originals.apply() # apply patches
+            self.active = True
+            self.model = shared.opts.sd_model_checkpoint
+        names, te_multipliers, unet_multipliers, dyn_dims = self.parse(p, params_list, step)
        networks.load_networks(names, te_multipliers, unet_multipliers, dyn_dims)
-        t2 = time.time()
-        if shared.opts.lora_add_hashes_to_infotext:
-            network_hashes = []
-            for item in networks.loaded_networks:
-                shorthash = item.network_on_disk.shorthash
-                if not shorthash:
-                    continue
-                alias = item.mentioned_name
-                if not alias:
-                    continue
-                alias = alias.replace(":", "").replace(",", "")
-                network_hashes.append(f"{alias}: {shorthash}")
-            if network_hashes:
-                p.extra_generation_params["Lora hashes"] = ", ".join(network_hashes)
-        if len(names) > 0 and step == 0:
-            shared.log.info(f'LoRA apply: {names} patch={t1-t0:.2f} load={t2-t1:.2f}')
-        elif self.active:
-            self.active = False
+        t1 = time.time()
+        if len(networks.loaded_networks) > 0 and step == 0:
+            self.infotext(p)
+            self.prompt(p)
+            shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} te={te_multipliers} unet={unet_multipliers} dims={dyn_dims} load={t1-t0:.2f}')

    def deactivate(self, p):
-        if shared.native and hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"):
-            if not (shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled is True):
-                try:
-                    if shared.opts.lora_fuse_diffusers:
-                        shared.sd_model.unfuse_lora()
-                    shared.sd_model.unload_lora_weights() # fails for non-CLIP models
-                    # shared.log.debug("LoRA unload")
-                except Exception:
-                    # shared.log.warning(f"LoRA unload: {e}")
-                    pass
-        if not self.active and getattr(networks, "originals", None ) is not None:
-            networks.originals.undo() # remove patches
-            if networks.debug:
-                shared.log.debug("LoRA deactivate")
+        t0 = time.time()
+        if shared.native and len(networks.diffuser_loaded) > 0:
+            if hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"):
+                if not (shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled is True):
+                    try:
+                        if shared.opts.lora_fuse_diffusers:
+                            shared.sd_model.unfuse_lora()
+                        shared.sd_model.unload_lora_weights() # fails for non-CLIP models
+                    except Exception:
+                        pass
+        t1 = time.time()
+        networks.timer['restore'] += t1 - t0
        if self.active and networks.debug:
-            shared.log.debug(f"LoRA end: load={networks.timer['load']:.2f} apply={networks.timer['apply']:.2f} restore={networks.timer['restore']:.2f}")
+            shared.log.debug(f"Network end: type=LoRA load={networks.timer['load']:.2f} apply={networks.timer['apply']:.2f} restore={networks.timer['restore']:.2f}")
        if self.errors:
            p.comment("Networks with errors: " + ", ".join(f"{k} ({v})" for k, v in self.errors.items()))
            for k, v in self.errors.items():
-                shared.log.error(f'LoRA errors: file="{k}" errors={v}')
+                shared.log.error(f'LoRA: name="{k}" errors={v}')
            self.errors.clear()
@@ -2,6 +2,7 @@ import os
 import re
 import bisect
 from typing import Dict
+import torch
 from modules import shared


@@ -173,7 +174,11 @@ class KeyConvert:
            map_key = map_keys[position - 1]
            if search_key.startswith(map_key):
                key = key.replace(map_key, self.UNET_CONVERSION_MAP[map_key]).replace("oft", "lora") # pylint: disable=unsubscriptable-object
+        if "lycoris" in key and "transformer" in key:
+            key = key.replace("lycoris", "lora_transformer")
        sd_module = shared.sd_model.network_layer_mapping.get(key, None)
+        if sd_module is None:
+            sd_module = shared.sd_model.network_layer_mapping.get(key.replace("guidance", "timestep"), None)  # FLUX1 fix
        # SegMoE begin
        expert_key = key + "_experts_0"
        expert_module = shared.sd_model.network_layer_mapping.get(expert_key, None)
@@ -253,3 +258,200 @@ def convert_diffusers_name_to_compvis(key, is_sd2):
        else:
            return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('self_attn', 'attn')}"
    return key
+
+
+# Taken from https://github.com/huggingface/diffusers/blob/main/src/diffusers/loaders/lora_conversion_utils.py
+# Modified from 'lora_A' and 'lora_B' to 'lora_down' and 'lora_up'
+# Added early exit
+# The utilities under `_convert_kohya_flux_lora_to_diffusers()`
+# are taken from https://github.com/kohya-ss/sd-scripts/blob/a61cf73a5cb5209c3f4d1a3688dd276a4dfd1ecb/networks/convert_flux_lora.py
+# All credits go to `kohya-ss`.
+def _convert_kohya_flux_lora_to_diffusers(state_dict):
+    def _convert_to_ai_toolkit(sds_sd, ait_sd, sds_key, ait_key):
+        if sds_key + ".lora_down.weight" not in sds_sd:
+            return
+        down_weight = sds_sd.pop(sds_key + ".lora_down.weight")
+
+        # scale weight by alpha and dim
+        rank = down_weight.shape[0]
+        alpha = sds_sd.pop(sds_key + ".alpha").item()  # alpha is scalar
+        scale = alpha / rank  # LoRA is scaled by 'alpha / rank' in forward pass, so we need to scale it back here
+
+        # calculate scale_down and scale_up to keep the same value. if scale is 4, scale_down is 2 and scale_up is 2
+        scale_down = scale
+        scale_up = 1.0
+        while scale_down * 2 < scale_up:
+            scale_down *= 2
+            scale_up /= 2
+
+        ait_sd[ait_key + ".lora_down.weight"] = down_weight * scale_down
+        ait_sd[ait_key + ".lora_up.weight"] = sds_sd.pop(sds_key + ".lora_up.weight") * scale_up
+
+    def _convert_to_ai_toolkit_cat(sds_sd, ait_sd, sds_key, ait_keys, dims=None):
+        if sds_key + ".lora_down.weight" not in sds_sd:
+            return
+        down_weight = sds_sd.pop(sds_key + ".lora_down.weight")
+        up_weight = sds_sd.pop(sds_key + ".lora_up.weight")
+        sd_lora_rank = down_weight.shape[0]
+
+        # scale weight by alpha and dim
+        alpha = sds_sd.pop(sds_key + ".alpha")
+        scale = alpha / sd_lora_rank
+
+        # calculate scale_down and scale_up
+        scale_down = scale
+        scale_up = 1.0
+        while scale_down * 2 < scale_up:
+            scale_down *= 2
+            scale_up /= 2
+
+        down_weight = down_weight * scale_down
+        up_weight = up_weight * scale_up
+
+        # calculate dims if not provided
+        num_splits = len(ait_keys)
+        if dims is None:
+            dims = [up_weight.shape[0] // num_splits] * num_splits
+        else:
+            assert sum(dims) == up_weight.shape[0]
+
+        # check upweight is sparse or not
+        is_sparse = False
+        if sd_lora_rank % num_splits == 0:
+            ait_rank = sd_lora_rank // num_splits
+            is_sparse = True
+            i = 0
+            for j in range(len(dims)):
+                for k in range(len(dims)):
+                    if j == k:
+                        continue
+                    is_sparse = is_sparse and torch.all(
+                        up_weight[i : i + dims[j], k * ait_rank : (k + 1) * ait_rank] == 0
+                    )
+                i += dims[j]
+            # if is_sparse:
+            #     print(f"weight is sparse: {sds_key}")
+
+        # make ai-toolkit weight
+        ait_down_keys = [k + ".lora_down.weight" for k in ait_keys]
+        ait_up_keys = [k + ".lora_up.weight" for k in ait_keys]
+        if not is_sparse:
+            # down_weight is copied to each split
+            ait_sd.update({k: down_weight for k in ait_down_keys})
+
+            # up_weight is split to each split
+            ait_sd.update({k: v for k, v in zip(ait_up_keys, torch.split(up_weight, dims, dim=0))})  # noqa: C416 # pylint: disable=unnecessary-comprehension
+        else:
+            # down_weight is chunked to each split
+            ait_sd.update({k: v for k, v in zip(ait_down_keys, torch.chunk(down_weight, num_splits, dim=0))})  # noqa: C416 # pylint: disable=unnecessary-comprehension
+
+            # up_weight is sparse: only non-zero values are copied to each split
+            i = 0
+            for j in range(len(dims)):
+                ait_sd[ait_up_keys[j]] = up_weight[i : i + dims[j], j * ait_rank : (j + 1) * ait_rank].contiguous()
+                i += dims[j]
+
+    def _convert_sd_scripts_to_ai_toolkit(sds_sd):
+        ait_sd = {}
+        for i in range(19):
+            _convert_to_ai_toolkit(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_double_blocks_{i}_img_attn_proj",
+                f"transformer.transformer_blocks.{i}.attn.to_out.0",
+            )
+            _convert_to_ai_toolkit_cat(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_double_blocks_{i}_img_attn_qkv",
+                [
+                    f"transformer.transformer_blocks.{i}.attn.to_q",
+                    f"transformer.transformer_blocks.{i}.attn.to_k",
+                    f"transformer.transformer_blocks.{i}.attn.to_v",
+                ],
+            )
+            _convert_to_ai_toolkit(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_double_blocks_{i}_img_mlp_0",
+                f"transformer.transformer_blocks.{i}.ff.net.0.proj",
+            )
+            _convert_to_ai_toolkit(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_double_blocks_{i}_img_mlp_2",
+                f"transformer.transformer_blocks.{i}.ff.net.2",
+            )
+            _convert_to_ai_toolkit(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_double_blocks_{i}_img_mod_lin",
+                f"transformer.transformer_blocks.{i}.norm1.linear",
+            )
+            _convert_to_ai_toolkit(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_double_blocks_{i}_txt_attn_proj",
+                f"transformer.transformer_blocks.{i}.attn.to_add_out",
+            )
+            _convert_to_ai_toolkit_cat(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_double_blocks_{i}_txt_attn_qkv",
+                [
+                    f"transformer.transformer_blocks.{i}.attn.add_q_proj",
+                    f"transformer.transformer_blocks.{i}.attn.add_k_proj",
+                    f"transformer.transformer_blocks.{i}.attn.add_v_proj",
+                ],
+            )
+            _convert_to_ai_toolkit(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_double_blocks_{i}_txt_mlp_0",
+                f"transformer.transformer_blocks.{i}.ff_context.net.0.proj",
+            )
+            _convert_to_ai_toolkit(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_double_blocks_{i}_txt_mlp_2",
+                f"transformer.transformer_blocks.{i}.ff_context.net.2",
+            )
+            _convert_to_ai_toolkit(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_double_blocks_{i}_txt_mod_lin",
+                f"transformer.transformer_blocks.{i}.norm1_context.linear",
+            )
+
+        for i in range(38):
+            _convert_to_ai_toolkit_cat(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_single_blocks_{i}_linear1",
+                [
+                    f"transformer.single_transformer_blocks.{i}.attn.to_q",
+                    f"transformer.single_transformer_blocks.{i}.attn.to_k",
+                    f"transformer.single_transformer_blocks.{i}.attn.to_v",
+                    f"transformer.single_transformer_blocks.{i}.proj_mlp",
+                ],
+                dims=[3072, 3072, 3072, 12288],
+            )
+            _convert_to_ai_toolkit(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_single_blocks_{i}_linear2",
+                f"transformer.single_transformer_blocks.{i}.proj_out",
+            )
+            _convert_to_ai_toolkit(
+                sds_sd,
+                ait_sd,
+                f"lora_unet_single_blocks_{i}_modulation_lin",
+                f"transformer.single_transformer_blocks.{i}.norm.linear",
+            )
+
+        if len(sds_sd) > 0:
+            return None
+
+        return ait_sd
+
+    return _convert_sd_scripts_to_ai_toolkit(state_dict)
@@ -0,0 +1,271 @@
+import os
+import time
+import json
+import datetime
+import torch
+from safetensors.torch import save_file
+import gradio as gr
+from rich import progress as p
+from modules import shared, devices
+from modules.ui_common import create_refresh_button
+from modules.call_queue import wrap_gradio_gpu_call
+
+
+class SVDHandler:
+    def __init__(self, maxrank=0, rank_ratio=1):
+        self.network_name: str = None
+        self.U: torch.Tensor = None
+        self.S: torch.Tensor = None
+        self.Vh: torch.Tensor = None
+        self.maxrank: int = maxrank
+        self.rank_ratio: float = rank_ratio
+        self.rank: int = 0
+        self.out_size: int = None
+        self.in_size: int = None
+        self.kernel_size: tuple[int, int] = None
+        self.conv2d: bool = False
+
+    def decompose(self, weight, backupweight):
+        self.conv2d = len(weight.size()) == 4
+        self.kernel_size = None if not self.conv2d else weight.size()[2:4]
+        self.out_size, self.in_size = weight.size()[0:2]
+        diffweight = weight.clone().to(devices.device)
+        diffweight -= backupweight.to(devices.device)
+        if self.conv2d:
+            if self.conv2d and self.kernel_size != (1, 1):
+                diffweight = diffweight.flatten(start_dim=1)
+            else:
+                diffweight = diffweight.squeeze()
+        self.U, self.S, self.Vh = torch.svd_lowrank(diffweight.to(device=devices.device, dtype=torch.float), self.maxrank, 2)
+        # del diffweight
+        self.U = self.U.to(device=devices.cpu, dtype=torch.bfloat16)
+        self.S = self.S.to(device=devices.cpu, dtype=torch.bfloat16)
+        self.Vh = self.Vh.t().to(device=devices.cpu, dtype=torch.bfloat16)  # svd_lowrank outputs a transposed matrix
+
+    def findrank(self):
+        if self.rank_ratio < 1:
+            S_squared = self.S.pow(2)
+            S_fro_sq = float(torch.sum(S_squared))
+            sum_S_squared = torch.cumsum(S_squared, dim=0) / S_fro_sq
+            index = int(torch.searchsorted(sum_S_squared, self.rank_ratio ** 2)) + 1
+            index = max(1, min(index, len(self.S) - 1))
+            self.rank = index
+            if self.maxrank > 0:
+                self.rank = min(self.rank, self.maxrank)
+        else:
+            self.rank = min(self.in_size, self.out_size, self.maxrank)
+
+    def makeweights(self):
+        self.findrank()
+        up = self.U[:, :self.rank] @ torch.diag(self.S[:self.rank])
+        down = self.Vh[:self.rank, :]
+        if self.conv2d and self.kernel_size is not None:
+            up = up.reshape(self.out_size, self.rank, 1, 1)
+            down = down.reshape(self.rank, self.in_size, self.kernel_size[0], self.kernel_size[1]) # pylint: disable=unsubscriptable-object
+        return_dict = {f'{self.network_name}.lora_up.weight': up.contiguous(),
+                       f'{self.network_name}.lora_down.weight': down.contiguous(),
+                       f'{self.network_name}.alpha': torch.tensor(down.shape[0]),
+                       }
+        return return_dict
+
+
+def loaded_lora():
+    if not shared.sd_loaded:
+        return ""
+    loaded = set()
+    if hasattr(shared.sd_model, 'unet'):
+        for _name, module in shared.sd_model.unet.named_modules():
+            current = getattr(module, "network_current_names", None)
+            if current is not None:
+                current = [item[0] for item in current]
+                loaded.update(current)
+    return list(loaded)
+
+
+def loaded_lora_str():
+    return ", ".join(loaded_lora())
+
+
+def make_meta(fn, maxrank, rank_ratio):
+    meta = {
+        "model_spec.sai_model_spec": "1.0.0",
+        "model_spec.title": os.path.splitext(os.path.basename(fn))[0],
+        "model_spec.author": "SD.Next",
+        "model_spec.implementation": "https://github.com/vladmandic/automatic",
+        "model_spec.date": datetime.datetime.now().astimezone().replace(microsecond=0).isoformat(),
+        "model_spec.base_model": shared.opts.sd_model_checkpoint,
+        "model_spec.dtype": str(devices.dtype),
+        "model_spec.base_lora": json.dumps(loaded_lora()),
+        "model_spec.config": f"maxrank={maxrank} rank_ratio={rank_ratio}",
+    }
+    if shared.sd_model_type == "sdxl":
+        meta["model_spec.architecture"] = "stable-diffusion-xl-v1-base/lora" # sai standard
+        meta["ss_base_model_version"] = "sdxl_base_v1-0" # kohya standard
+    elif shared.sd_model_type == "sd":
+        meta["model_spec.architecture"] = "stable-diffusion-v1/lora"
+        meta["ss_base_model_version"] = "sd_v1"
+    elif shared.sd_model_type == "f1":
+        meta["model_spec.architecture"] = "flux-1-dev/lora"
+        meta["ss_base_model_version"] = "flux1"
+    elif shared.sd_model_type == "sc":
+        meta["model_spec.architecture"] = "stable-cascade-v1-prior/lora"
+    return meta
+
+
+def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite):
+    if not shared.sd_loaded or not shared.native:
+        msg = "LoRA extract: model not loaded"
+        shared.log.warning(msg)
+        yield msg
+        return
+    if loaded_lora() == "":
+        msg = "LoRA extract: no LoRA detected"
+        shared.log.warning(msg)
+        yield msg
+        return
+    if not fn:
+        msg = "LoRA extract: target filename required"
+        shared.log.warning(msg)
+        yield msg
+        return
+    t0 = time.time()
+    maxrank = int(maxrank)
+    rank_ratio = 1 if not auto_rank else rank_ratio
+    shared.log.debug(f'LoRA extract: modules={modules} maxrank={maxrank} auto={auto_rank} ratio={rank_ratio} fn="{fn}"')
+    shared.state.begin('LoRA extract')
+
+    with p.Progress(p.TextColumn('[cyan]LoRA extract'), p.BarColumn(), p.TaskProgressColumn(), p.TimeRemainingColumn(), p.TimeElapsedColumn(), p.TextColumn('[cyan]{task.description}'), console=shared.console) as progress:
+
+        if 'te' in modules and getattr(shared.sd_model, 'text_encoder', None) is not None:
+            modules = shared.sd_model.text_encoder.named_modules()
+            task = progress.add_task(description="te1 decompose", total=len(list(modules)))
+            for name, module in shared.sd_model.text_encoder.named_modules():
+                progress.update(task, advance=1)
+                weights_backup = getattr(module, "network_weights_backup", None)
+                if weights_backup is None or getattr(module, "network_current_names", None) is None:
+                    continue
+                prefix = "lora_te1_" if hasattr(shared.sd_model, 'text_encoder_2') else "lora_te_"
+                module.svdhandler = SVDHandler(maxrank, rank_ratio)
+                module.svdhandler.network_name = prefix + name.replace(".", "_")
+                with devices.inference_context():
+                    module.svdhandler.decompose(module.weight, weights_backup)
+            progress.remove_task(task)
+        t1 = time.time()
+
+        if 'te' in modules and getattr(shared.sd_model, 'text_encoder_2', None) is not None:
+            modules = shared.sd_model.text_encoder_2.named_modules()
+            task = progress.add_task(description="te2 decompose", total=len(list(modules)))
+            for name, module in shared.sd_model.text_encoder_2.named_modules():
+                progress.update(task, advance=1)
+                weights_backup = getattr(module, "network_weights_backup", None)
+                if weights_backup is None or getattr(module, "network_current_names", None) is None:
+                    continue
+                module.svdhandler = SVDHandler(maxrank, rank_ratio)
+                module.svdhandler.network_name = "lora_te2_" + name.replace(".", "_")
+                with devices.inference_context():
+                    module.svdhandler.decompose(module.weight, weights_backup)
+            progress.remove_task(task)
+        t2 = time.time()
+
+        if 'unet' in modules and getattr(shared.sd_model, 'unet', None) is not None:
+            modules = shared.sd_model.unet.named_modules()
+            task = progress.add_task(description="unet decompose", total=len(list(modules)))
+            for name, module in shared.sd_model.unet.named_modules():
+                progress.update(task, advance=1)
+                weights_backup = getattr(module, "network_weights_backup", None)
+                if weights_backup is None or getattr(module, "network_current_names", None) is None:
+                    continue
+                module.svdhandler = SVDHandler(maxrank, rank_ratio)
+                module.svdhandler.network_name = "lora_unet_" + name.replace(".", "_")
+                with devices.inference_context():
+                    module.svdhandler.decompose(module.weight, weights_backup)
+            progress.remove_task(task)
+        t3 = time.time()
+
+        # TODO: Handle quant for Flux
+        # if 'te' in modules and getattr(shared.sd_model, 'transformer', None) is not None:
+        #     for name, module in shared.sd_model.transformer.named_modules():
+        #         if "norm" in name and "linear" not in name:
+        #             continue
+        #         weights_backup = getattr(module, "network_weights_backup", None)
+        #         if weights_backup is None:
+        #             continue
+        #         module.svdhandler = SVDHandler()
+        #         module.svdhandler.network_name = "lora_transformer_" + name.replace(".", "_")
+        #         module.svdhandler.decompose(module.weight, weights_backup)
+        #         module.svdhandler.findrank(rank, rank_ratio)
+
+        lora_state_dict = {}
+        for sub in ['text_encoder', 'text_encoder_2', 'unet', 'transformer']:
+            submodel = getattr(shared.sd_model, sub, None)
+            if submodel is not None:
+                modules = submodel.named_modules()
+                task = progress.add_task(description=f"{sub} exctract", total=len(list(modules)))
+                for _name, module in submodel.named_modules():
+                    progress.update(task, advance=1)
+                    if not hasattr(module, "svdhandler"):
+                        continue
+                    lora_state_dict.update(module.svdhandler.makeweights())
+                    del module.svdhandler
+                progress.remove_task(task)
+        t4 = time.time()
+
+    if not os.path.isabs(fn):
+        fn = os.path.join(shared.cmd_opts.lora_dir, fn)
+    if not fn.endswith('.safetensors'):
+        fn += '.safetensors'
+    if os.path.exists(fn):
+        if overwrite:
+            os.remove(fn)
+        else:
+            msg = f'LoRA extract: fn="{fn}" file exists'
+            shared.log.warning(msg)
+            yield msg
+            return
+
+    shared.state.end()
+    meta = make_meta(fn, maxrank, rank_ratio)
+    shared.log.debug(f'LoRA metadata: {meta}')
+    try:
+        save_file(tensors=lora_state_dict, metadata=meta, filename=fn)
+    except Exception as e:
+        msg = f'LoRA extract error: fn="{fn}" {e}'
+        shared.log.error(msg)
+        yield msg
+        return
+    t5 = time.time()
+    shared.log.debug(f'LoRA extract: time={t5-t0:.2f} te1={t1-t0:.2f} te2={t2-t1:.2f} unet={t3-t2:.2f} save={t5-t4:.2f}')
+    keys = list(lora_state_dict.keys())
+    msg = f'LoRA extract: fn="{fn}" keys={len(keys)}'
+    shared.log.info(msg)
+    yield msg
+
+
+def create_ui():
+    def gr_show(visible=True):
+        return {"visible": visible, "__type__": "update"}
+
+    with gr.Tab(label="Extract LoRA"):
+        with gr.Row():
+            loaded = gr.Textbox(placeholder="Press refresh to query loaded LoRA", label="Loaded LoRA", interactive=False)
+            create_refresh_button(loaded, lambda: None, lambda: {'value': loaded_lora_str()}, "testid")
+        with gr.Group():
+            with gr.Row():
+                modules = gr.CheckboxGroup(label="Modules to extract", value=['unet'], choices=['te', 'unet'])
+            with gr.Row():
+                auto_rank = gr.Checkbox(value=False, label="Automatically determine rank")
+                rank_ratio = gr.Slider(label="Autorank ratio", value=1, minimum=0, maximum=1, step=0.05, visible=False)
+                rank = gr.Slider(label="Maximum rank", value=32, minimum=1, maximum=256)
+        with gr.Row():
+            filename = gr.Textbox(label="LoRA target filename")
+            overwrite = gr.Checkbox(value=False, label="Overwrite existing file")
+        with gr.Row():
+            extract = gr.Button(value="Extract LoRA", variant='primary')
+            status = gr.HTML(value="", show_label=False)
+
+        auto_rank.change(fn=lambda x: gr_show(x), inputs=[auto_rank], outputs=[rank_ratio])
+        extract.click(
+            fn=wrap_gradio_gpu_call(make_lora, extra_outputs=[]),
+            inputs=[filename, rank, auto_rank, rank_ratio, modules, overwrite],
+            outputs=[status]
+        )
@@ -1,6 +1,7 @@
+import sys
 import torch
 import networks
-from modules import patches, shared
+from modules import patches, shared, model_quant


 class LoraPatches:
@@ -16,14 +17,32 @@ class LoraPatches:
        self.LayerNorm_load_state_dict = None
        self.MultiheadAttention_forward = None
        self.MultiheadAttention_load_state_dict = None
+        # optional quant forwards
+        self.Linear4bit_forward = None # bitsandbytes
+        self.QLinear_forward = None # optimum.quanto
+        self.QConv2d_forward = None # optimum.quanto
+
+    def handle_quant(self, apply: bool):
+        if 'bitsandbytes' in sys.modules: # lora should not be first to initialize quantization
+            bnb = model_quant.load_bnb(silent=True)
+            if bnb is not None:
+                if apply:
+                    self.Linear4bit_forward = patches.patch(__name__, bnb.nn.Linear4bit, 'forward', networks.network_Linear4bit_forward)
+                else:
+                    self.Linear4bit_forward = patches.undo(__name__, bnb.nn.Linear4bit, 'forward') # pylint: disable=E1128
+        if 'optimum.quanto' in sys.modules:
+            quanto = model_quant.load_quanto(silent=True)
+            if quanto is not None:
+                if apply:
+                    self.QLinear_forward = patches.patch(__name__, quanto.nn.QLinear, 'forward', networks.network_QLinear_forward)
+                    self.QConv2d_forward = patches.patch(__name__, quanto.nn.QConv2d, 'forward', networks.network_QConv2d_forward)
+                else:
+                    self.QLinear_forward = patches.undo(__name__, quanto.nn.QLinear, 'forward') # pylint: disable=E1128
+                    self.QConv2d_forward = patches.undo(__name__, quanto.nn.QConv2d, 'forward') # pylint: disable=E1128

    def apply(self):
        if self.active or shared.opts.lora_force_diffusers:
            return
-        if "Model" in shared.opts.optimum_quanto_weights or "Text Encoder" in shared.opts.optimum_quanto_weights:
-            from optimum import quanto
-            self.QLinear_forward = patches.patch(__name__, quanto.nn.QLinear, 'forward', networks.network_QLinear_forward) # pylint: disable=attribute-defined-outside-init
-            self.QConv2d_forward = patches.patch(__name__, quanto.nn.QConv2d, 'forward', networks.network_QConv2d_forward) # pylint: disable=attribute-defined-outside-init
        self.Linear_forward = patches.patch(__name__, torch.nn.Linear, 'forward', networks.network_Linear_forward)
        self.Linear_load_state_dict = patches.patch(__name__, torch.nn.Linear, '_load_from_state_dict', networks.network_Linear_load_state_dict)
        self.Conv2d_forward = patches.patch(__name__, torch.nn.Conv2d, 'forward', networks.network_Conv2d_forward)
@@ -34,6 +53,7 @@ class LoraPatches:
        self.LayerNorm_load_state_dict = patches.patch(__name__, torch.nn.LayerNorm, '_load_from_state_dict', networks.network_LayerNorm_load_state_dict)
        self.MultiheadAttention_forward = patches.patch(__name__, torch.nn.MultiheadAttention, 'forward', networks.network_MultiheadAttention_forward)
        self.MultiheadAttention_load_state_dict = patches.patch(__name__, torch.nn.MultiheadAttention, '_load_from_state_dict', networks.network_MultiheadAttention_load_state_dict)
+        self.handle_quant(apply=True)
        networks.timer['load'] = 0
        networks.timer['apply'] = 0
        networks.timer['restore'] = 0
@@ -42,10 +62,6 @@ class LoraPatches:
    def undo(self):
        if not self.active or shared.opts.lora_force_diffusers:
            return
-        if "Model" in shared.opts.optimum_quanto_weights or "Text Encoder" in shared.opts.optimum_quanto_weights:
-            from optimum import quanto
-            self.QLinear_forward = patches.undo(__name__, quanto.nn.QLinear, 'forward') # pylint: disable=E1128, attribute-defined-outside-init
-            self.QConv2d_forward = patches.undo(__name__, quanto.nn.QConv2d, 'forward') # pylint: disable=E1128, attribute-defined-outside-init
        self.Linear_forward = patches.undo(__name__, torch.nn.Linear, 'forward') # pylint: disable=E1128
        self.Linear_load_state_dict = patches.undo(__name__, torch.nn.Linear, '_load_from_state_dict') # pylint: disable=E1128
        self.Conv2d_forward = patches.undo(__name__, torch.nn.Conv2d, 'forward') # pylint: disable=E1128
@@ -56,5 +72,6 @@ class LoraPatches:
        self.LayerNorm_load_state_dict = patches.undo(__name__, torch.nn.LayerNorm, '_load_from_state_dict') # pylint: disable=E1128
        self.MultiheadAttention_forward = patches.undo(__name__, torch.nn.MultiheadAttention, 'forward') # pylint: disable=E1128
        self.MultiheadAttention_load_state_dict = patches.undo(__name__, torch.nn.MultiheadAttention, '_load_from_state_dict') # pylint: disable=E1128
+        self.handle_quant(apply=False)
        patches.originals.pop(__name__, None)
        self.active = False
@@ -13,7 +13,10 @@ class SdVersion(enum.Enum):
    Unknown = 1
    SD1 = 2
    SD2 = 3
+    SD3 = 3
    SDXL = 4
+    SC = 5
+    F1 = 6


 class NetworkOnDisk:
@@ -40,13 +43,38 @@ class NetworkOnDisk:
        self.sd_version = self.detect_version()

    def detect_version(self):
-        if str(self.metadata.get('ss_base_model_version', "")).startswith("sdxl_"):
-            return SdVersion.SDXL
-        elif str(self.metadata.get('ss_v2', "")) == "True":
-            return SdVersion.SD2
-        elif len(self.metadata):
-            return SdVersion.SD1
-        return SdVersion.Unknown
+        base = str(self.metadata.get('ss_base_model_version', "")).lower()
+        arch = str(self.metadata.get('modelspec.architecture', "")).lower()
+        if base.startswith("sd_v1"):
+            return 'sd1'
+        if base.startswith("sdxl"):
+            return 'xl'
+        if base.startswith("stable_cascade"):
+            return 'sc'
+        if base.startswith("sd3"):
+            return 'sd3'
+        if base.startswith("flux"):
+            return 'f1'
+
+        if arch.startswith("stable-diffusion-v1"):
+            return 'sd1'
+        if arch.startswith("stable-diffusion-xl"):
+            return 'xl'
+        if arch.startswith("stable-cascade"):
+            return 'sc'
+        if arch.startswith("flux"):
+            return 'f1'
+
+        if "v1-5" in str(self.metadata.get('ss_sd_model_name', "")):
+            return 'sd1'
+        if str(self.metadata.get('ss_v2', "")) == "True":
+            return 'sd2'
+        if 'flux' in self.name.lower():
+            return 'f1'
+        if 'xl' in self.name.lower():
+            return 'xl'
+
+        return ''

    def set_hash(self, v):
        self.hash = v or ''
@@ -72,6 +100,7 @@ class Network:  # LoraModule
        self.bundle_embeddings = {}
        self.mtime = None
        self.mentioned_name = None
+        self.tags = None
        """the text that was used to add the network to prompt - can be either name or an alias"""


@@ -5,7 +5,6 @@ class ModuleTypeFull(network.ModuleType):
    def create_module(self, net: network.Network, weights: network.NetworkWeights):
        if all(x in weights.w for x in ["diff"]):
            return NetworkModuleFull(net, weights)
-
        return None


@@ -5,7 +5,6 @@ class ModuleTypeIa3(network.ModuleType):
    def create_module(self, net: network.Network, weights: network.NetworkWeights):
        if all(x in weights.w for x in ["weight"]):
            return NetworkModuleIa3(net, weights)
-
        return None


@@ -13,6 +13,7 @@ class ModuleTypeLora(network.ModuleType):


 class NetworkModuleLora(network.NetworkModule):
+
    def __init__(self,  net: network.Network, weights: network.NetworkWeights):
        super().__init__(net, weights)
        self.up_model = self.create_module(weights.w, "lora_up.weight")
@@ -21,11 +22,12 @@ class NetworkModuleLora(network.NetworkModule):
        self.dim = weights.w["lora_down.weight"].shape[0]

    def create_module(self, weights, key, none_ok=False):
+        from modules.shared import opts
        weight = weights.get(key)
        if weight is None and none_ok:
            return None
        linear_modules = [torch.nn.Linear, torch.nn.modules.linear.NonDynamicallyQuantizableLinear, torch.nn.MultiheadAttention, diffusers_lora.LoRACompatibleLinear]
-        is_linear = type(self.sd_module) in linear_modules or self.sd_module.__class__.__name__ in {"NNCFLinear", "QLinear"}
+        is_linear = type(self.sd_module) in linear_modules or self.sd_module.__class__.__name__ in {"NNCFLinear", "QLinear", "Linear4bit"}
        is_conv = type(self.sd_module) in [torch.nn.Conv2d, diffusers_lora.LoRACompatibleConv] or self.sd_module.__class__.__name__ in {"NNCFConv2d", "QConv2d"}
        if is_linear:
            weight = weight.reshape(weight.shape[0], -1)
@@ -47,17 +49,19 @@ class NetworkModuleLora(network.NetworkModule):
            if weight.shape != module.weight.shape:
                weight = weight.reshape(module.weight.shape)
            module.weight.copy_(weight)
-        module.to(device=devices.cpu, dtype=devices.dtype)
+        if opts.lora_load_gpu:
+            module = module.to(device=devices.device, dtype=devices.dtype)
        module.weight.requires_grad_(False)
        return module

    def calc_updown(self, target): # pylint: disable=W0237
-        up = self.up_model.weight.to(target.device, dtype=target.dtype)
-        down = self.down_model.weight.to(target.device, dtype=target.dtype)
+        target_dtype = target.dtype if target.dtype != torch.uint8 else self.up_model.weight.dtype
+        up = self.up_model.weight.to(target.device, dtype=target_dtype)
+        down = self.down_model.weight.to(target.device, dtype=target_dtype)
        output_shape = [up.size(0), down.size(1)]
        if self.mid_model is not None:
            # cp-decomposition
-            mid = self.mid_model.weight.to(target.device, dtype=target.dtype)
+            mid = self.mid_model.weight.to(target.device, dtype=target_dtype)
            updown = lyco_helpers.rebuild_cp_decomposition(up, down, mid)
            output_shape += mid.shape[2:]
        else:
@@ -71,5 +75,4 @@ class NetworkModuleLora(network.NetworkModule):
        self.down_model.to(device=devices.device)
        if hasattr(y, "scale"):
            return y(scale=1) + self.up_model(self.down_model(x)) * self.multiplier() * self.calc_scale()
-
        return y + self.up_model(self.down_model(x)) * self.multiplier() * self.calc_scale()
@@ -8,19 +8,15 @@ class ModuleTypeOFT(network.ModuleType):
    def create_module(self, net: network.Network, weights: network.NetworkWeights):
        if all(x in weights.w for x in ["oft_blocks"]) or all(x in weights.w for x in ["oft_diag"]):
            return NetworkModuleOFT(net, weights)
-
        return None

 # Supports both kohya-ss' implementation of COFT  https://github.com/kohya-ss/sd-scripts/blob/main/networks/oft.py
 # and KohakuBlueleaf's implementation of OFT/COFT https://github.com/KohakuBlueleaf/LyCORIS/blob/dev/lycoris/modules/diag_oft.py
 class NetworkModuleOFT(network.NetworkModule): # pylint: disable=abstract-method
    def __init__(self,  net: network.Network, weights: network.NetworkWeights):
-
        super().__init__(net, weights)
-
        self.lin_module = None
        self.org_module: list[torch.Module] = [self.sd_module]
-
        self.scale = 1.0

        # kohya-ss
@@ -30,7 +30,6 @@ force_models = [ # forced always
    'kandinsky',
    'hunyuandit',
    'auraflow',
-    'f1',
 ]

 force_classes = [ # forced always
@@ -17,7 +17,7 @@ import network_overrides
 import lora_convert
 import torch
 import diffusers.models.lora
-from modules import shared, devices, sd_models, sd_models_compile, errors, scripts, files_cache
+from modules import shared, devices, sd_models, sd_models_compile, errors, scripts, files_cache, model_quant


 debug = os.environ.get('SD_LORA_DEBUG', None) is not None
@@ -26,7 +26,7 @@ extra_network_lora = None
 available_networks = {}
 available_network_aliases = {}
 loaded_networks: List[network.Network] = []
-timer = { 'load': 0, 'apply': 0, 'restore': 0 }
+timer = { 'load': 0, 'apply': 0, 'restore': 0, 'deactivate': 0 }
 # networks_in_memory = {}
 lora_cache = {}
 diffuser_loaded = []
@@ -48,25 +48,33 @@ convert_diffusers_name_to_compvis = lora_convert.convert_diffusers_name_to_compv


 def assign_network_names_to_compvis_modules(sd_model):
+    if sd_model is None:
+        return
    network_layer_mapping = {}
    if shared.native:
-        if not hasattr(shared.sd_model, 'text_encoder') or not hasattr(shared.sd_model, 'unet'):
-            sd_model.network_layer_mapping = {}
-            return
-        for name, module in shared.sd_model.text_encoder.named_modules():
-            prefix = "lora_te1_" if shared.sd_model_type == "sdxl" else "lora_te_"
-            network_name = prefix + name.replace(".", "_")
-            network_layer_mapping[network_name] = module
-            module.network_layer_name = network_name
-        if shared.sd_model_type == "sdxl":
+        if hasattr(shared.sd_model, 'text_encoder') and shared.sd_model.text_encoder is not None:
+            for name, module in shared.sd_model.text_encoder.named_modules():
+                prefix = "lora_te1_" if hasattr(shared.sd_model, 'text_encoder_2') else "lora_te_"
+                network_name = prefix + name.replace(".", "_")
+                network_layer_mapping[network_name] = module
+                module.network_layer_name = network_name
+        if hasattr(shared.sd_model, 'text_encoder_2'):
            for name, module in shared.sd_model.text_encoder_2.named_modules():
                network_name = "lora_te2_" + name.replace(".", "_")
                network_layer_mapping[network_name] = module
                module.network_layer_name = network_name
-        for name, module in shared.sd_model.unet.named_modules():
-            network_name = "lora_unet_" + name.replace(".", "_")
-            network_layer_mapping[network_name] = module
-            module.network_layer_name = network_name
+        if hasattr(shared.sd_model, 'unet'):
+            for name, module in shared.sd_model.unet.named_modules():
+                network_name = "lora_unet_" + name.replace(".", "_")
+                network_layer_mapping[network_name] = module
+                module.network_layer_name = network_name
+        if hasattr(shared.sd_model, 'transformer'):
+            for name, module in shared.sd_model.transformer.named_modules():
+                network_name = "lora_transformer_" + name.replace(".", "_")
+                network_layer_mapping[network_name] = module
+                if "norm" in network_name and "linear" not in network_name:
+                    continue
+                module.network_layer_name = network_name
    else:
        if not hasattr(shared.sd_model, 'cond_stage_model'):
            sd_model.network_layer_mapping = {}
@@ -82,26 +90,28 @@ def assign_network_names_to_compvis_modules(sd_model):
    sd_model.network_layer_mapping = network_layer_mapping


-def load_diffusers(name, network_on_disk, lora_scale=1.0) -> network.Network:
+def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_default_multiplier) -> network.Network:
    t0 = time.time()
    name = name.replace(".", "_")
    #cached = lora_cache.get(name, None)
-    shared.log.debug(f'LoRA load: name="{name}" file="{network_on_disk.filename}" type=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_diffusers}')
+    shared.log.debug(f'Load network: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_diffusers}')
    # if cached is not None:
    #    return cached
    if not shared.native:
        return None
    if not hasattr(shared.sd_model, 'load_lora_weights'):
-        shared.log.error(f"LoRA load failed: class={shared.sd_model.__class__} does not implement load lora")
+        shared.log.error(f'Load network: type=LoRA class={shared.sd_model.__class__} does not implement load lora')
        return None
    try:
        shared.sd_model.load_lora_weights(network_on_disk.filename, adapter_name=name)
    except Exception as e:
        if 'already in use' in str(e):
-            # shared.log.warning(f"LoRA load failed: file={network_on_disk.filename} {e}")
            pass
        else:
-            shared.log.error(f"LoRA load failed: file={network_on_disk.filename} {e}")
+            if 'The following keys have not been correctly renamed' in str(e):
+                shared.log.error(f'Load network: type=LoRA name="{name}" diffusers unsupported format')
+            else:
+                shared.log.error(f'Load network: type=LoRA name="{name}" {e}')
            if debug:
                errors.display(e, "LoRA")
            return None
@@ -120,12 +130,14 @@ def load_network(name, network_on_disk) -> network.Network:
    t0 = time.time()
    cached = lora_cache.get(name, None)
    if debug:
-        shared.log.debug(f'LoRA load: name="{name}" file="{network_on_disk.filename}" type=lora {"cached" if cached else ""}')
+        shared.log.debug(f'Load network: type=LoRA name="{name}" file="{network_on_disk.filename}" type=lora {"cached" if cached else ""}')
    if cached is not None:
        return cached
    net = network.Network(name, network_on_disk)
    net.mtime = os.path.getmtime(network_on_disk.filename)
-    sd = sd_models.read_state_dict(network_on_disk.filename)
+    sd = sd_models.read_state_dict(network_on_disk.filename, what='network')
+    if shared.sd_model_type == 'f1':  # if kohya flux lora, convert state_dict
+        sd = lora_convert._convert_kohya_flux_lora_to_diffusers(sd) or sd  # pylint: disable=protected-access
    assign_network_names_to_compvis_modules(shared.sd_model) # this should not be needed but is here as an emergency fix for an unknown error people are experiencing in 1.2.0
    keys_failed_to_match = {}
    matched_networks = {}
@@ -145,8 +157,6 @@ def load_network(name, network_on_disk) -> network.Network:
            network_part = '.'.join(parts[-2:]).replace('lora_A', 'lora_down').replace('lora_B', 'lora_up')
        else:
            key_network_without_network_parts, network_part = key_network.split(".", 1)
-        # if debug:
-        #     shared.log.debug(f'LoRA load: name="{name}" full={key_network} network={network_part} key={key_network_without_network_parts}')
        key, sd_module = convert(key_network_without_network_parts)  # Now returns lists
        if sd_module[0] is None:
            if "bundle_emb" not in key_network:
@@ -156,22 +166,26 @@ def load_network(name, network_on_disk) -> network.Network:
            if k not in matched_networks:
                matched_networks[k] = network.NetworkWeights(network_key=key_network, sd_key=k, w={}, sd_module=module)
            matched_networks[k].w[network_part] = weight
+    network_types = []
    for key, weights in matched_networks.items():
        net_module = None
        for nettype in module_types:
            net_module = nettype.create_module(net, weights)
            if net_module is not None:
+                network_types.append(nettype.__class__.__name__)
                break
        if net_module is None:
            shared.log.error(f'LoRA unhandled: name={name} key={key} weights={weights.w.keys()}')
        else:
            net.modules[key] = net_module
    if len(keys_failed_to_match) > 0:
-        shared.log.warning(f"LoRA file={network_on_disk.filename} unmatched={len(keys_failed_to_match)} matched={len(matched_networks)}")
+        shared.log.warning(f'LoRA name="{name}" type={set(network_types)} unmatched={len(keys_failed_to_match)} matched={len(matched_networks)}')
        if debug:
-            shared.log.debug(f"LoRA file={network_on_disk.filename} unmatched={keys_failed_to_match}")
-    elif debug:
-        shared.log.debug(f"LoRA file={network_on_disk.filename} unmatched={len(keys_failed_to_match)} matched={len(matched_networks)}")
+            shared.log.debug(f'LoRA name="{name}" unmatched={keys_failed_to_match}')
+    else:
+        shared.log.debug(f'LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)}')
+    if len(matched_networks) == 0:
+        return None
    lora_cache[name] = net
    t1 = time.time()
    net.bundle_embeddings = bundle_embeddings
@@ -180,19 +194,16 @@ def load_network(name, network_on_disk) -> network.Network:


 def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None):
-    if shared.opts.diffusers_offload_mode == "balanced":
-        sd_models.disable_offload(shared.sd_model)
-        sd_models.move_model(shared.sd_model, devices.cpu)
-    networks_on_disk = [available_network_aliases.get(name, None) for name in names]
+    networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names]
    if any(x is None for x in networks_on_disk):
        list_available_networks()
-        networks_on_disk = [available_network_aliases.get(name, None) for name in names]
+        networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names]
    failed_to_load_networks = []
    recompile_model = False
    if shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled:
        if len(names) == len(shared.compiled_model_state.lora_model):
            for i, name in enumerate(names):
-                if shared.compiled_model_state.lora_model[i] != f"{name}:{te_multipliers[i] if te_multipliers else 1.0}":
+                if shared.compiled_model_state.lora_model[i] != f"{name}:{te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier}":
                    recompile_model = True
                    shared.compiled_model_state.lora_model = []
                    break
@@ -218,48 +229,48 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No
        if network_on_disk is not None:
            shorthash = getattr(network_on_disk, 'shorthash', '').lower()
            if debug:
-                shared.log.debug(f'LoRA load: name="{name}" file="{network_on_disk.filename}" hash="{shorthash}"')
+                shared.log.debug(f'Load network: type=LoRA name="{name}" file="{network_on_disk.filename}" hash="{shorthash}"')
            try:
                if recompile_model:
-                    shared.compiled_model_state.lora_model.append(f"{name}:{te_multipliers[i] if te_multipliers else 1.0}")
+                    shared.compiled_model_state.lora_model.append(f"{name}:{te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier}")
                if shared.native and (shared.opts.lora_force_diffusers or network_overrides.check_override(shorthash)): # OpenVINO only works with Diffusers LoRa loading
-                    net = load_diffusers(name, network_on_disk, lora_scale=te_multipliers[i] if te_multipliers else 1.0)
+                    net = load_diffusers(name, network_on_disk, lora_scale=te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier)
                else:
                    net = load_network(name, network_on_disk)
                if net is not None:
                    net.mentioned_name = name
                    network_on_disk.read_hash()
            except Exception as e:
-                shared.log.error(f"LoRA load failed: file={network_on_disk.filename} {e}")
+                shared.log.error(f'Load network: type=LoRA file="{network_on_disk.filename}" {e}')
                if debug:
-                    errors.display(e, f"LoRA load failed file={network_on_disk.filename}")
+                    errors.display(e, 'LoRA')
                continue
        if net is None:
            failed_to_load_networks.append(name)
-            shared.log.error(f"LoRA unknown type: network={name}")
+            shared.log.error(f'Load network: type=LoRA name="{name}" detected={network_on_disk.sd_version if network_on_disk is not None else None} failed')
            continue
        if shared.native:
            shared.sd_model.embedding_db.load_diffusers_embedding(None, net.bundle_embeddings)
-        net.te_multiplier = te_multipliers[i] if te_multipliers else 1.0
-        net.unet_multiplier = unet_multipliers[i] if unet_multipliers else 1.0
-        net.dyn_dim = dyn_dims[i] if dyn_dims else 1.0
+        net.te_multiplier = te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier
+        net.unet_multiplier = unet_multipliers[i] if unet_multipliers else shared.opts.extra_networks_default_multiplier
+        net.dyn_dim = dyn_dims[i] if dyn_dims else shared.opts.extra_networks_default_multiplier
        loaded_networks.append(net)

    while len(lora_cache) > shared.opts.lora_in_memory_limit:
        name = next(iter(lora_cache))
        lora_cache.pop(name, None)
    if len(diffuser_loaded) > 0:
-        shared.log.debug(f'LoRA loaded={diffuser_loaded} scales={diffuser_scales}')
+        shared.log.debug(f'Load network: type=LoRA loaded={diffuser_loaded} scales={diffuser_scales}')
        shared.sd_model.set_adapters(adapter_names=diffuser_loaded, adapter_weights=diffuser_scales)
        if shared.opts.lora_fuse_diffusers:
-            shared.sd_model.fuse_lora(adapter_names=diffuser_loaded, lora_scale=1.0, fuse_unet=True, fuse_text_encoder=True)
+            shared.sd_model.fuse_lora(adapter_names=diffuser_loaded, lora_scale=1.0, fuse_unet=True, fuse_text_encoder=True) # fuse uses fixed scale since later apply does the scaling
            shared.sd_model.unload_lora_weights()
    if len(loaded_networks) > 0 and debug:
-        shared.log.debug(f'LoRA loaded={len(loaded_networks)} cache={list(lora_cache)}')
+        shared.log.debug(f'Load network: type=LoRA loaded={len(loaded_networks)} cache={list(lora_cache)}')
    devices.torch_gc()

    if recompile_model:
-        shared.log.info("LoRA recompiling model")
+        shared.log.info("Load network: type=LoRA recompiling model")
        backup_lora_model = shared.compiled_model_state.lora_model
        if 'Model' in shared.opts.cuda_compile:
            shared.sd_model = sd_models_compile.compile_diffusers(shared.sd_model)
@@ -274,6 +285,8 @@ def network_restore_weights_from_backup(self: Union[torch.nn.Conv2d, torch.nn.Li
    weights_backup = getattr(self, "network_weights_backup", None)
    bias_backup = getattr(self, "network_bias_backup", None)
    if weights_backup is None and bias_backup is None:
+        t1 = time.time()
+        timer['restore'] += t1 - t0
        return
    # if debug:
    #     shared.log.debug('LoRA restore weights')
@@ -284,6 +297,14 @@ def network_restore_weights_from_backup(self: Union[torch.nn.Conv2d, torch.nn.Li
        elif hasattr(self, "qweight") and hasattr(self, "freeze"):
            self.weight = torch.nn.Parameter(weights_backup.to(self.weight.device, copy=True))
            self.freeze()
+        elif getattr(self, "quant_type", None) in ['nf4', 'fp4']:
+            bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True)
+            if bnb is not None:
+                device = self.weight.device
+                self.weight = bnb.nn.Params4bit(weights_backup, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize)
+                self.weight.to(device)
+            else:
+                self.weight.copy_(weights_backup)
        else:
            self.weight.copy_(weights_backup)
    if bias_backup is not None:
@@ -300,6 +321,35 @@ def network_restore_weights_from_backup(self: Union[torch.nn.Conv2d, torch.nn.Li
    timer['restore'] += t1 - t0


+def maybe_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, torch.nn.MultiheadAttention, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], wanted_names, current_names): # pylint: disable=W0613
+    weights_backup = getattr(self, "network_weights_backup", None)
+    if weights_backup is None and wanted_names != (): # pylint: disable=C1803
+        if isinstance(self, torch.nn.MultiheadAttention):
+            weights_backup = (self.in_proj_weight.clone().to(devices.cpu), self.out_proj.weight.clone().to(devices.cpu))
+        elif getattr(self.weight, "quant_type", None) in ['nf4', 'fp4']:
+            bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True)
+            if bnb is not None:
+                with devices.inference_context():
+                    weights_backup = bnb.functional.dequantize_4bit(self.weight, quant_state=self.weight.quant_state, quant_type=self.weight.quant_type, blocksize=self.weight.blocksize,).to(devices.cpu)
+                    self.quant_state = self.weight.quant_state
+                    self.quant_type = self.weight.quant_type
+                    self.blocksize = self.weight.blocksize
+            else:
+                weights_backup = self.weight.clone().to(devices.cpu)
+        else:
+            weights_backup = self.weight.clone().to(devices.cpu)
+        self.network_weights_backup = weights_backup
+    bias_backup = getattr(self, "network_bias_backup", None)
+    if bias_backup is None:
+        if isinstance(self, torch.nn.MultiheadAttention) and self.out_proj.bias is not None:
+            bias_backup = self.out_proj.bias.clone().to(devices.cpu)
+        elif getattr(self, 'bias', None) is not None:
+            bias_backup = self.bias.clone().to(devices.cpu)
+        else:
+            bias_backup = None
+        self.network_bias_backup = bias_backup
+
+
 def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, torch.nn.MultiheadAttention, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv]):
    """
    Applies the currently selected set of networks to the weights of torch layer self.
@@ -312,25 +362,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
    t0 = time.time()
    current_names = getattr(self, "network_current_names", ())
    wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks)
-    weights_backup = getattr(self, "network_weights_backup", None)
-    if weights_backup is None and wanted_names != (): # pylint: disable=C1803
-        if current_names != ():
-            raise RuntimeError("no backup weights found and current weights are not unchanged")
-        if isinstance(self, torch.nn.MultiheadAttention):
-            weights_backup = (self.in_proj_weight.to(devices.cpu, copy=True), self.out_proj.weight.to(devices.cpu, copy=True))
-        else:
-            weights_backup = self.weight.to(devices.cpu, copy=True)
-        self.network_weights_backup = weights_backup
-    bias_backup = getattr(self, "network_bias_backup", None)
-    if bias_backup is None:
-        if isinstance(self, torch.nn.MultiheadAttention) and self.out_proj.bias is not None:
-            bias_backup = self.out_proj.bias.to(devices.cpu, copy=True)
-        elif getattr(self, 'bias', None) is not None:
-            bias_backup = self.bias.to(devices.cpu, copy=True)
-        else:
-            bias_backup = None
-        self.network_bias_backup = bias_backup
-
+    if any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 # pylint: disable=R1729
+        maybe_backup_weights(self, wanted_names, current_names)
    if current_names != wanted_names:
        network_restore_weights_from_backup(self)
        for net in loaded_networks:
@@ -344,7 +377,17 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
                        if len(weight.shape) == 4 and weight.shape[1] == 9:
                            # inpainting model. zero pad updown to make channel[1]  4 to 9
                            updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable
-                        self.weight = torch.nn.Parameter(weight + updown)
+                        if getattr(self.weight, "quant_type", None) in ['nf4', 'fp4']: # or self.weight.numel() != updown.numel():
+                            bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True)
+                            if bnb is not None:
+                                device = self.weight.device
+                                weight = bnb.functional.dequantize_4bit(self.weight, quant_state=self.weight.quant_state, quant_type=self.weight.quant_type, blocksize=self.weight.blocksize)
+                                self.weight = bnb.nn.Params4bit(weight + updown, quant_state=self.quant_state, quant_type=shared.opts.lora_quant.lower(), blocksize=self.blocksize)
+                                self.weight.to(device)
+                            else:
+                                self.weight = torch.nn.Parameter(weight + updown)
+                        else:
+                            self.weight = torch.nn.Parameter(weight + updown)
                        if hasattr(self, "qweight") and hasattr(self, "freeze"):
                            self.freeze()
                        if ex_bias is not None and hasattr(self, 'bias'):
@@ -356,8 +399,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
                    extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1
                    if debug:
                        module_name = net.modules.get(network_layer_name, None)
-                        shared.log.error(f"LoRA apply weight name={net.name} module={module_name} layer={network_layer_name} {e}")
-                        errors.display(e, 'LoRA apply weight')
+                        shared.log.error(f'LoRA apply weight name="{net.name}" module="{module_name}" layer="{network_layer_name}" {e}')
+                        errors.display(e, 'LoRA')
                        raise RuntimeError('LoRA apply weight') from e
                continue
            # alternative workflow looking at _*_proj layers
@@ -382,12 +425,12 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
                            self.out_proj.bias += ex_bias
                except RuntimeError as e:
                    if debug:
-                        shared.log.debug(f"LoRA network={net.name} layer={network_layer_name} {e}")
+                        shared.log.debug(f'LoRA network="{net.name}" layer="{network_layer_name}" {e}')
                    extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1
                continue
            if module is None:
                continue
-            shared.log.warning(f"LoRA network={net.name} layer={network_layer_name} unsupported operation")
+            shared.log.warning(f'LoRA network="{net.name}" layer="{network_layer_name}" unsupported operation')
            extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1
        self.network_current_names = wanted_names
    t1 = time.time()
@@ -420,15 +463,11 @@ def network_reset_cached_weight(self: Union[torch.nn.Conv2d, torch.nn.Linear]):


 def network_Linear_forward(self, input): # pylint: disable=W0622
-    if shared.opts.lora_functional:
-        return network_forward(self, input, originals.Linear_forward)
    network_apply_weights(self)
    return originals.Linear_forward(self, input)


 def network_QLinear_forward(self, input): # pylint: disable=W0622
-    if shared.opts.lora_functional:
-        return network_forward(self, input, originals.Linear_forward)
    network_apply_weights(self)
    return torch.nn.functional.linear(input, self.qweight, bias=self.bias)

@@ -438,16 +477,20 @@ def network_Linear_load_state_dict(self, *args, **kwargs):
    return originals.Linear_load_state_dict(self, *args, **kwargs)


+def network_Linear4bit_forward(self, input): # pylint: disable=W0622
+    network_apply_weights(self)
+    return originals.Linear4bit_forward(self, input)
+#
+# def network_Linear4bit_load_state_dict(self, *args, **kwargs):
+#     network_reset_cached_weight(self)
+#     return originals.Linear4bit_load_state_dict(self, *args, **kwargs)
+
 def network_Conv2d_forward(self, input): # pylint: disable=W0622
-    if shared.opts.lora_functional:
-        return network_forward(self, input, originals.Conv2d_forward)
    network_apply_weights(self)
    return originals.Conv2d_forward(self, input)


 def network_QConv2d_forward(self, input): # pylint: disable=W0622
-    if shared.opts.lora_functional:
-        return network_forward(self, input, originals.Conv2d_forward)
    network_apply_weights(self)
    return self._conv_forward(input, self.qweight, self.bias) # pylint: disable=protected-access

@@ -458,8 +501,6 @@ def network_Conv2d_load_state_dict(self, *args, **kwargs):


 def network_GroupNorm_forward(self, input): # pylint: disable=W0622
-    if shared.opts.lora_functional:
-        return network_forward(self, input, originals.GroupNorm_forward)
    network_apply_weights(self)
    return originals.GroupNorm_forward(self, input)

@@ -470,8 +511,6 @@ def network_GroupNorm_load_state_dict(self, *args, **kwargs):


 def network_LayerNorm_forward(self, input): # pylint: disable=W0622
-    if shared.opts.lora_functional:
-        return network_forward(self, input, originals.LayerNorm_forward)
    network_apply_weights(self)
    return originals.LayerNorm_forward(self, input)

@@ -509,6 +548,7 @@ def list_available_networks():
        if not os.path.isfile(filename):
            return
        name = os.path.splitext(os.path.basename(filename))[0]
+        name = name.replace('.', '_')
        try:
            entry = network.NetworkOnDisk(name, filename)
            available_networks[entry.name] = entry
@@ -521,13 +561,13 @@ def list_available_networks():
            if entry.shorthash:
                available_network_hash_lookup[entry.shorthash] = entry
        except OSError as e:  # should catch FileNotFoundError and PermissionError etc.
-            shared.log.error(f"Failed to load network {name} from {filename} {e}")
+            shared.log.error(f'LoRA: filename="{filename}" {e}')

    candidates = list(files_cache.list_files(*directories, ext_filter=[".pt", ".ckpt", ".safetensors"]))
    with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor:
        for fn in candidates:
            executor.submit(add_network, fn)
-    shared.log.info(f'LoRA networks: available={len(available_networks)} folders={len(forbidden_network_aliases)}')
+    shared.log.info(f'Available LoRAs: items={len(available_networks)} folders={len(forbidden_network_aliases)}')


 def infotext_pasted(infotext, params): # pylint: disable=W0613
@@ -1,10 +1,11 @@
 import re
 import networks
 import lora # pylint: disable=unused-import
+from lora_extract import create_ui
 from network import NetworkOnDisk
 from ui_extra_networks_lora import ExtraNetworksPageLora
 from extra_networks_lora import ExtraNetworkLora
-from modules import script_callbacks, ui_extra_networks, extra_networks
+from modules import script_callbacks, extra_networks, ui_extra_networks, ui_models # pylint: disable=unused-import


 re_lora = re.compile("<lora:([^:]+):")
@@ -14,6 +15,7 @@ def before_ui():
    ui_extra_networks.register_page(ExtraNetworksPageLora())
    networks.extra_network_lora = ExtraNetworkLora()
    extra_networks.register_extra_network(networks.extra_network_lora)
+    ui_models.extra_ui.append(create_ui)


 def create_lora_json(obj: NetworkOnDisk):
@@ -1,11 +1,13 @@
 import os
 import json
 import concurrent
-import network
 import networks
 from modules import shared, ui_extra_networks


+debug = os.environ.get('SD_LOAD_DEBUG', None) is not None
+
+
 class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
    def __init__(self):
        super().__init__('Lora')
@@ -16,22 +18,12 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):

    def create_item(self, name):
        l = networks.available_networks.get(name)
+        if l is None:
+            shared.log.warning(f'Networks: type=lora registered={len(list(networks.available_networks))} file="{name}" not registered')
+            return None
        try:
            # path, _ext = os.path.splitext(l.filename)
            name = os.path.splitext(os.path.relpath(l.filename, shared.cmd_opts.lora_dir))[0]
-            if not shared.native:
-                if l.sd_version == network.SdVersion.SDXL:
-                    return None
-            elif shared.native:
-                if shared.sd_model_type == 'none': # return all when model is not loaded
-                    pass
-                elif shared.sd_model_type == 'sdxl':
-                    if l.sd_version == network.SdVersion.SD1 or l.sd_version == network.SdVersion.SD2:
-                        return None
-                elif shared.sd_model_type == 'sd':
-                    if l.sd_version == network.SdVersion.SDXL:
-                        return None
-
            item = {
                "type": 'Lora',
                "name": name,
@@ -41,21 +33,24 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
                "metadata": json.dumps(l.metadata, indent=4) if l.metadata else None,
                "mtime": os.path.getmtime(l.filename),
                "size": os.path.getsize(l.filename),
+                "version": l.sd_version,
            }
            info = self.find_info(l.filename)

            tags = {}
-            possible_tags = l.metadata.get('ss_tag_frequency', {}) if l.metadata is not None else {} # tags from model metedata
-            if isinstance(possible_tags, str):
-                possible_tags = {}
-            for k, v in possible_tags.items():
-                words = k.split('_', 1) if '_' in k else [v, k]
-                words = [str(w).replace('.json', '') for w in words]
-                if words[0] == '{}':
-                    words[0] = 0
-                tag = ' '.join(words[1:]).lower()
-                tags[tag] = words[0]
-
+            if l.metadata is not None:
+                modelspec_tags = l.metadata.get('modelspec.tags', {})
+                possible_tags = l.metadata.get('ss_tag_frequency', {}) # tags from model metedata
+                possible_tags.update(modelspec_tags)
+                if isinstance(possible_tags, str):
+                    possible_tags = {}
+                for k, v in possible_tags.items():
+                    words = k.split('_', 1) if '_' in k else [v, k]
+                    words = [str(w).replace('.json', '') for w in words]
+                    if words[0] == '{}':
+                        words[0] = 0
+                    tag = ' '.join(words[1:]).lower()
+                    tags[tag] = words[0]

            def find_version():
                found_versions = []
@@ -69,7 +64,6 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
                    found_versions = all_versions
                return found_versions

-            find_version()
            for v in find_version():  # trigger words from info json
                possible_tags = v.get('trainedWords', [])
                if isinstance(possible_tags, list):
@@ -102,9 +96,10 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):

            return item
        except Exception as e:
-            shared.log.debug(f"Networks error: type=lora file={name} {e}")
-            from modules import errors
-            errors.display('e', 'Lora')
+            shared.log.error(f'Networks: type=lora file="{name}" {e}')
+            if debug:
+                from modules import errors
+                errors.display('e', 'Lora')
            return None

    def list_items(self):
@@ -119,11 +119,11 @@
  {"id":"","label":"Steps","localized":"","hint":"How many times to improve the generated image iteratively; higher values take longer; very low values can produce bad results"},
  {"id":"","label":"Tiling","localized":"","hint":"Produce an image that can be tiled"},
  {"id":"","label":"full quality","localized":"","hint":"Use full quality VAE to decode latent samples"},
-  {"id":"","label":"face restore","localized":"","hint":"Run processed image through additional face restoration model"},
+  {"id":"","label":"detailer","localized":"","hint":"Run processed image through additional detailer model"},
  {"id":"","label":"hidiffusion","localized":"","hint":"HiDiffusion allows creation of high-resolution images using your standard models without duplicates/distortions and improved performance"},
  {"id":"","label":"HDR Clamp","localized":"","hint":"Adjusts the level of nonsensical details by pruning values that deviate significantly from the distribution mean. It is particularly useful for enhancing generation at higher guidance scales, identifying outliers early in the process and applying mathematical adjustments based on the Range (Boundary) and Threshold settings. Think of it as setting the range within which you want your image values to be, and adjusting the threshold determines which values should be brought back into that range"},
  {"id":"","label":"HDR Maximize","localized":"","hint":"Calculates a 'normalization factor' by dividing the maximum tensor value by the specified range multiplied by 4. This factor is then used to shift the channels within the given boundary, ensuring maximum dynamic range for subsequent processing. The objective is to optimize dynamic range for external applications like Photoshop, particularly for adjusting levels, contrast, and brightness"},
-  {"id":"","label":"Enable second pass","localized":"","hint":"Use a similar process as image to image to upscale and/or add detail to the final image. Optionally uses refiner model to enhance image details."},
+  {"id":"","label":"Enable refine pass","localized":"","hint":"Use a similar process as image to image to upscale and/or add detail to the final image. Optionally uses refiner model to enhance image details."},
  {"id":"","label":"Denoising strength","localized":"","hint":"Determines how little respect the algorithm should have for image's content. At 0, nothing will change, and at 1 you'll get an unrelated image. With values below 1.0, processing will take less steps than the Sampling Steps slider specifies"},
  {"id":"","label":"Denoise start","localized":"","hint":"Override denoise strength by stating how early base model should finish and when refiner should start. Only applicable to refiner usage. If set to 0 or 1, denoising strength will be used"},
  {"id":"","label":"Hires steps","localized":"","hint":"Number of sampling steps for upscaled picture. If 0, uses same as for original"},
@@ -132,13 +132,13 @@
  {"id":"","label":"Force Hires","localized":"","hint":"Hires runs automatically when Latent upscale is selected, but its skipped when using non-latent upscalers. Enable force hires to run hires with non-latent upscalers"},
  {"id":"","label":"Resize width to","localized":"","hint":"Resizes image to this width. If 0, width is inferred from either of two nearby sliders"},
  {"id":"","label":"Resize height to","localized":"","hint":"Resizes image to this height. If 0, height is inferred from either of two nearby sliders"},
-  {"id":"","label":"Secondary sampler","localized":"","hint":"Use specific sampler as fallback sampler if primary is not supported for specific operation"},
+  {"id":"","label":"Refine sampler","localized":"","hint":"Use specific sampler as fallback sampler if primary is not supported for specific operation"},
  {"id":"","label":"Refiner start","localized":"","hint":"Refiner pass will start when base model is this much complete (set to larger than 0 and smaller than 1 to run after full base model run)"},
  {"id":"","label":"Refiner steps","localized":"","hint":"Number of steps to use for refiner pass"},
-  {"id":"","label":"Secondary CFG Scale","localized":"","hint":"CFG scale used for refiner pass"},
+  {"id":"","label":"Refine CFG Scale","localized":"","hint":"CFG scale used for refiner pass"},
  {"id":"","label":"Rescale guidance","localized":"","hint":"Rescale CFG generated noise to avoid overexposed images"},
-  {"id":"","label":"Secondary Prompt","localized":"","hint":"Prompt used for both second encoder in base model (if it exists) and for refiner pass (if enabled)"},
-  {"id":"","label":"Secondary negative prompt","localized":"","hint":"Negative prompt used for both second encoder in base model (if it exists) and for refiner pass (if enabled)"},
+  {"id":"","label":"Refine Prompt","localized":"","hint":"Prompt used for both second encoder in base model (if it exists) and for refiner pass (if enabled)"},
+  {"id":"","label":"Refine negative prompt","localized":"","hint":"Negative prompt used for both second encoder in base model (if it exists) and for refiner pass (if enabled)"},
  {"id":"","label":"Width","localized":"","hint":"Image width"},
  {"id":"","label":"Height","localized":"","hint":"Image height"},
  {"id":"","label":"Batch count","localized":"","hint":"How many batches of images to create (has no impact on generation performance or VRAM usage)"},
@@ -179,7 +179,7 @@
  {"id":"","label":"Show result images","localized":"","hint":"Enable to show the processed images in the image pane"},
  {"id":"","label":"Resize","localized":"","hint":"Resizing details. Higher resolutions require additional processing memory."},
  {"id":"","label":"Crop to fit","localized":"","hint":"If the dimensions of your source image (e.g. 512x510) deviate from your target dimensions (e.g. 1024x768) this function will fit your upscaled image into your target size image. Excess will be cropped"},
-  {"id":"","label":"Secondary Upscaler","localized":"","hint":"Select secondary upscaler to run after initial upscaler"},
+  {"id":"","label":"Refine Upscaler","localized":"","hint":"Select secondary upscaler to run after initial upscaler"},
  {"id":"","label":"Upscaler 2 visibility","localized":"","hint":"Strength of the secondary upscaler"}
 ],
 "models tabs": [
@@ -1,21 +1,28 @@
 {
+  "Tempest SD-XL v0.1": {
+    "path": "TempestV0.1-Artistic.safetensors@https://huggingface.co/dataautogpt3/TempestV0.1/resolve/main/TempestV0.1-Artistic.safetensors?download=true",
+    "preview": "TempestV0.1-Artistic.jpg",
+    "desc": "The TempestV0.1 Initiative is a powerhouse in image generation, leveraging an unparalleled dataset of over 6 million images. The collection's vast scale, with resolutions from 1400x2100 to 4800x7200, encompasses 200GB of high-quality content.",
+    "extras": "width: 2048, height: 1024, sampler: DEIS, steps: 40, cfg_scale: 6.0"
+  },
+
  "Juggernaut SD-XL XI": {
    "path": "juggernautXL_juggXIByRundiffusion.safetensors@https://civitai.com/api/download/models/782002",
    "preview": "juggernautXL_v9Rundiffusionphoto2.jpg",
    "desc": "Showcase finetuned model based on Stable diffusion XL",
-    "extras": "width: 1024, height: 1024, sampler: DEIS, steps: 20, cfg_scale: 6.0"
+    "extras": "sampler: DEIS, steps: 20, cfg_scale: 6.0"
  },
  "Juggernaut SD-XL X Hyper": {
    "path": "Juggernaut_X_RunDiffusion_Hyper.safetensors@https://civitai.com/api/download/models/471120",
    "preview": "juggernautXL_v9Rundiffusionphoto2.jpg",
    "desc": "Showcase finetuned model based on Stable diffusion XL",
-    "extras": "width: 1024, height: 1024, sampler: DEIS, steps: 20, cfg_scale: 6.0"
+    "extras": "sampler: DEIS, steps: 20, cfg_scale: 6.0"
  },
  "Juggernaut SD-XL IX Lightning": {
    "path": "juggernautXL_v9Rdphoto2Lightning.safetensors@https://civitai.com/api/download/models/357609",
    "preview": "juggernautXL_v9Rdphoto2Lightning.jpg",
    "desc": "Showcase finetuned model based on Stable diffusion XL",
-    "extras": "width: 1024, height: 1024, sampler: DPM SDE, steps: 6, cfg_scale: 2.0"
+    "extras": "sampler: DPM SDE, steps: 6, cfg_scale: 2.0"
  },
  "Juggernaut SD Reborn": {
    "original": true,
@@ -42,16 +49,9 @@
    "path": "dreamshaperXL_v21TurboDPMSDE.safetensors@https://civitai.com/api/download/models/351306",
    "preview": "dreamshaperXL_v21TurboDPMSDE.jpg",
    "desc": "Showcase finetuned model based on Stable diffusion XL",
-    "extras": "width: 1024, height: 1024, sampler: DPM SDE, steps: 8, cfg_scale: 2.0"
+    "extras": "sampler: DPM SDE, steps: 8, cfg_scale: 2.0"
  }, 

-  "Tempest SD-XL v0.1": {
-    "path": "TempestV0.1-Artistic.safetensors@https://huggingface.co/dataautogpt3/TempestV0.1/resolve/main/TempestV0.1-Artistic.safetensors?download=true",
-    "preview": "TempestV0.1-Artistic.jpg",
-    "desc": "The TempestV0.1 Initiative is a powerhouse in image generation, leveraging an unparalleled dataset of over 6 million images. The collection's vast scale, with resolutions from 1400x2100 to 4800x7200, encompasses 200GB of high-quality content.",
-    "extras": "width: 2048, height: 1024, sampler: DEIS, steps: 40, cfg_scale: 6.0"
-  },
-
  "SDXS DreamShaper 512": {
    "path": "IDKiro/sdxs-512-dreamshaper",
    "preview": "IDKiro--sdxs-512-dreamshaper.jpg",
@@ -93,7 +93,7 @@
    "path": "sd_xl_base_1.0.safetensors@https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0.safetensors?download=true",
    "preview": "sd_xl_base_1.0.jpg",
    "desc": "Stable Diffusion XL (SDXL) is the latest AI image generation model that is tailored towards more photorealistic outputs with more detailed imagery and composition compared to previous SD models, including SD 2.1. It can make realistic faces, legible text within the images, and better image composition, all while using shorter and simpler prompts at a greatly increased base resolution of 1024x1024. Just like its predecessors, SDXL has the ability to generate image variations using image-to-image prompting, inpainting (reimagining of the selected parts of an image), and outpainting (creating new parts that lie outside the image borders).",
-    "extras": "width: 1024, height: 1024, sampler: DEIS, steps: 20, cfg_scale: 6.0"
+    "extras": "sampler: DEIS, steps: 20, cfg_scale: 6.0"
  },
  "StabilityAI Stable Cascade": {
    "path": "huggingface/stabilityai/stable-cascade",
@@ -101,7 +101,7 @@
    "variant": "bf16",
    "desc": "Stable Cascade is a diffusion model built upon the Würstchen architecture and its main difference to other models like Stable Diffusion is that it is working at a much smaller latent space. Why is this important? The smaller the latent space, the faster you can run inference and the cheaper the training becomes. How small is the latent space? Stable Diffusion uses a compression factor of 8, resulting in a 1024x1024 image being encoded to 128x128. Stable Cascade achieves a compression factor of 42, meaning that it is possible to encode a 1024x1024 image to 24x24, while maintaining crisp reconstructions. The text-conditional model is then trained in the highly compressed latent space. Previous versions of this architecture, achieved a 16x cost reduction over Stable Diffusion 1.5",
    "preview": "stabilityai--stable-cascade.jpg",
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0"
+    "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0"
  },
  "StabilityAI Stable Cascade Lite": {
    "path": "huggingface/stabilityai/stable-cascade-lite",
@@ -109,16 +109,31 @@
    "variant": "bf16",
    "desc": "Stable Cascade is a diffusion model built upon the Würstchen architecture and its main difference to other models like Stable Diffusion is that it is working at a much smaller latent space. Why is this important? The smaller the latent space, the faster you can run inference and the cheaper the training becomes. How small is the latent space? Stable Diffusion uses a compression factor of 8, resulting in a 1024x1024 image being encoded to 128x128. Stable Cascade achieves a compression factor of 42, meaning that it is possible to encode a 1024x1024 image to 24x24, while maintaining crisp reconstructions. The text-conditional model is then trained in the highly compressed latent space. Previous versions of this architecture, achieved a 16x cost reduction over Stable Diffusion 1.5",
    "preview": "stabilityai--stable-cascade.jpg",
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0"
+    "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0"
  },
  "StabilityAI Stable Diffusion 3 Medium": {
-    "path": "huggingface/stabilityai/stable-diffusion-3-medium-diffusers",
+    "path": "stabilityai/stable-diffusion-3-medium-diffusers",
    "skip": true,
    "variant": "fp16",
-    "te3": null,
    "desc": "Stable Diffusion 3 Medium is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features greatly improved performance in image quality, typography, complex prompt understanding, and resource-efficiency",
    "preview": "stabilityai--stable-diffusion-3.jpg",
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 7.0"
+    "extras": "sampler: Default, cfg_scale: 7.0"
+  },
+  "StabilityAI Stable Diffusion 3.5 Large": {
+    "path": "stabilityai/stable-diffusion-3.5-large",
+    "skip": true,
+    "variant": "fp16",
+    "desc": "Stable Diffusion 3 Medium is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features greatly improved performance in image quality, typography, complex prompt understanding, and resource-efficiency",
+    "preview": "stabilityai--stable-diffusion-3_5.jpg",
+    "extras": "sampler: Default, cfg_scale: 7.0"
+  },
+  "StabilityAI Stable Diffusion 3.5 Turbo": {
+    "path": "stabilityai/stable-diffusion-3.5-large-turbo",
+    "skip": true,
+    "variant": "fp16",
+    "desc": "Stable Diffusion 3 Medium is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features greatly improved performance in image quality, typography, complex prompt understanding, and resource-efficiency",
+    "preview": "stabilityai--stable-diffusion-3_5.jpg",
+    "extras": "sampler: Default, cfg_scale: 7.0"
  },

  "Black Forest Labs FLUX.1 Dev": {
@@ -126,43 +141,49 @@
    "preview": "black-forest-labs--FLUX.1-dev.jpg",
    "desc": "FLUX.1 models are based on a hybrid architecture of multimodal and parallel diffusion transformer blocks, scaled to 12B parameters and builing on flow matching",
    "skip": true,
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 3.5"
+    "extras": "sampler: Default, cfg_scale: 3.5"
  },
  "Black Forest Labs FLUX.1 Schnell": {
    "path": "black-forest-labs/FLUX.1-schnell",
    "preview": "black-forest-labs--FLUX.1-schnell.jpg",
    "desc": "FLUX.1 models are based on a hybrid architecture of multimodal and parallel diffusion transformer blocks, scaled to 12B parameters and builing on flow matching. Trained using latent adversarial diffusion distillation, FLUX.1 [schnell] can generate high-quality images in only 1 to 4 steps",
    "skip": true,
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 3.5"
+    "extras": "sampler: Default, cfg_scale: 3.5"
  },
  "Black Forest Labs FLUX.1 Dev qint8": {
    "path": "Disty0/FLUX.1-dev-qint8",
    "preview": "black-forest-labs--FLUX.1-dev.jpg",
    "desc": "FLUX.1 models are based on a hybrid architecture of multimodal and parallel diffusion transformer blocks, scaled to 12B parameters and builing on flow matching",
    "skip": true,
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 3.5"
+    "extras": "sampler: Default, cfg_scale: 3.5"
  },
  "Black Forest Labs FLUX.1 Dev qint4": {
    "path": "Disty0/FLUX.1-dev-qint4",
    "preview": "black-forest-labs--FLUX.1-dev.jpg",
    "desc": "FLUX.1 models are based on a hybrid architecture of multimodal and parallel diffusion transformer blocks, scaled to 12B parameters and builing on flow matching",
    "skip": true,
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 3.5"
+    "extras": "sampler: Default, cfg_scale: 3.5"
  },
  "Black Forest Labs FLUX.1 Dev nf4": {
    "path": "sayakpaul/flux.1-dev-nf4",
    "preview": "black-forest-labs--FLUX.1-dev.jpg",
    "desc": "FLUX.1 models are based on a hybrid architecture of multimodal and parallel diffusion transformer blocks, scaled to 12B parameters and builing on flow matching",
    "skip": true,
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 3.5"
+    "extras": "sampler: Default, cfg_scale: 3.5"
  },

+  "VectorSpaceLab OmniGen v1": {
+    "path": "Shitao/OmniGen-v1",
+    "desc": "OmniGen is a unified image generation model that can generate a wide range of images from multi-modal prompts. It is designed to be simple, flexible and easy to use.",
+    "preview": "Shitao--OmniGen-v1.jpg",
+    "skip": true
+  }, 
+
  "AuraFlow 0.3": {
    "path": "fal/AuraFlow-v0.3",
    "desc": "AuraFlow v0.3 is the fully open-sourced flow-based text-to-image generation model. The model was trained with more compute compared to the previous version, AuraFlow-v0.2. Compared to AuraFlow-v0.2, the model is fine-tuned on more aesthetic datasets and now supports various aspect ratio, (now width and height up to 1536 pixels).",
    "preview": "fal--AuraFlow-v0.3.jpg",
-    "skip": true,
-    "extras": "width: 1024, height: 1024"
+    "skip": true
  }, 

  "Segmind Vega": {
@@ -171,7 +192,7 @@
    "desc": "The Segmind-Vega Model is a distilled version of the Stable Diffusion XL (SDXL), offering a remarkable 70% reduction in size and an impressive 100% speedup while retaining high-quality text-to-image generation capabilities. Trained on diverse datasets, including Grit and Midjourney scrape data, it excels at creating a wide range of visual content based on textual prompts. Employing a knowledge distillation strategy, Segmind-Vega leverages the teachings of several expert models, including SDXL, ZavyChromaXL, and JuggernautXL, to combine their strengths and produce compelling visual outputs.",
    "variant": "fp16",
    "skip": true,
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 9.0"
+    "extras": "sampler: Default, cfg_scale: 9.0"
  },
  "Segmind SSD-1B": {
    "path": "huggingface/segmind/SSD-1B",
@@ -179,7 +200,7 @@
    "desc": "The Segmind Stable Diffusion Model (SSD-1B) offers a compact, efficient, and distilled version of the SDXL model. At 50% smaller and 60% faster than Stable Diffusion XL (SDXL), it provides quick and seamless performance without sacrificing image quality.",
    "variant": "fp16",
    "skip": true,
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 9.0"
+    "extras": "sampler: Default, cfg_scale: 9.0"
  },
  "Segmind Tiny": {
    "path": "segmind/tiny-sd",
@@ -197,7 +218,7 @@
    "path": "segmind/SegMoE-4x2-v0",
    "preview": "segmind--SegMoE-4x2-v0.jpg",
    "desc": "SegMoE-4x2-v0 is an untrained Segmind Mixture of Diffusion Experts Model generated using segmoe from 4 Expert SDXL models. SegMoE is a powerful framework for dynamically combining Stable Diffusion Models into a Mixture of Experts within minutes without training",
-    "extras": "width: 1024, height: 1024, sampler: Default"
+    "extras": "sampler: Default"
  },

  "Pixart-α XL 2 Medium": {
@@ -210,7 +231,7 @@
    "path": "PixArt-alpha/PixArt-XL-2-1024-MS",
    "desc": "PixArt-α is a Transformer-based T2I diffusion model whose image generation quality is competitive with state-of-the-art image generators (e.g., Imagen, SDXL, and even Midjourney), and the training speed markedly surpasses existing large-scale T2I models. Extensive experiments demonstrate that PIXART-α excels in image quality, artistry, and semantic control. It can directly generate 1024px images from text prompts within a single sampling process.",
    "preview": "PixArt-alpha--PixArt-XL-2-1024-MS.jpg",
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 2.0"
+    "extras": "sampler: Default, cfg_scale: 2.0"
  },
  "Pixart-Σ Small": {
    "path": "huggingface/PixArt-alpha/PixArt-Sigma-XL-2-512-MS",
@@ -224,21 +245,21 @@
    "desc": "PixArt-Σ, a Diffusion Transformer model (DiT) capable of directly generating images at 4K resolution. PixArt-Σ represents a significant advancement over its predecessor, PixArt-α, offering images of markedly higher fidelity and improved alignment with text prompts.",
    "preview": "PixArt-alpha--pixart_sigma_sdxlvae_T5_diffusers.jpg",
    "skip": true,
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 2.0"
+    "extras": "sampler: Default, cfg_scale: 2.0"
  },
  "Pixart-Σ Large": {
    "path": "huggingface/PixArt-alpha/PixArt-Sigma-XL-2-2K-MS",
    "desc": "PixArt-Σ, a Diffusion Transformer model (DiT) capable of directly generating images at 4K resolution. PixArt-Σ represents a significant advancement over its predecessor, PixArt-α, offering images of markedly higher fidelity and improved alignment with text prompts.",
    "preview": "PixArt-alpha--pixart_sigma_sdxlvae_T5_diffusers.jpg",
    "skip": true,
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 2.0"
+    "extras": "sampler: Default, cfg_scale: 2.0"
  },
  
  "Tencent HunyuanDiT 1.2": {
    "path": "Tencent-Hunyuan/HunyuanDiT-v1.2-Diffusers",
    "desc": "Hunyuan-DiT : A Powerful Multi-Resolution Diffusion Transformer with Fine-Grained Chinese Understanding.",
    "preview": "Tencent-Hunyuan--HunyuanDiT-v1.2-Diffusers.jpg",
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 2.0"
+    "extras": "sampler: Default, cfg_scale: 2.0"
  },

  "AlphaVLLM Lumina Next SFT": {
@@ -246,7 +267,7 @@
    "desc": "The Lumina-Next-SFT is a Next-DiT model containing 2B parameters and utilizes Gemma-2B as the text encoder, enhanced through high-quality supervised fine-tuning (SFT).",
    "preview": "Alpha-VLLM--Lumina-Next-SFT-diffusers.jpg",
    "skip": true,
-    "extras": "width: 1024, height: 1024, sampler: Default"
+    "extras": "sampler: Default"
  }, 

  "Kwai Kolors": {
@@ -274,7 +295,7 @@
    "desc": "Kandinsky 3.0 is an open-source text-to-image diffusion model built upon the Kandinsky2-x model family. In comparison to its predecessors, Kandinsky 3.0 incorporates more data and specifically related to Russian culture, which allows to generate pictures related to Russin culture. Furthermore, enhancements have been made to the text understanding and visual quality of the model, achieved by increasing the size of the text encoder and Diffusion U-Net models, respectively.",
    "preview": "kandinsky-community--kandinsky-3.jpg",
    "variant": "fp16",
-    "extras": "width: 1024, height: 1024, sampler: Default"
+    "extras": "sampler: Default"
  },

  "Playground v1": {
@@ -299,13 +320,26 @@
    "path": "playgroundai/playground-v2-1024px-aesthetic",
    "desc": "Playground v2 is a diffusion-based text-to-image generative model. The model was trained from scratch by the research team at Playground. Images generated by Playground v2 are favored 2.5 times more than those produced by Stable Diffusion XL, according to Playground’s user study.",
    "preview": "playgroundai--playground-v2-1024px-aesthetic.jpg",
-    "extras": "width: 1024, height: 1024, sampler: Default"
+    "extras": "sampler: Default"
  },
  "Playground v2.5": {
    "path": "playground-v2.5-1024px-aesthetic.fp16.safetensors@https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic/resolve/main/playground-v2.5-1024px-aesthetic.fp16.safetensors?download=true",
    "desc": "Playground v2.5 is a diffusion-based text-to-image generative model, and a successor to Playground v2. Playground v2.5 is the state-of-the-art open-source model in aesthetic quality. Our user studies demonstrate that our model outperforms SDXL, Playground v2, PixArt-α, DALL-E 3, and Midjourney 5.2.",
    "preview": "playgroundai--playground-v2-1024px-aesthetic.jpg",
-    "extras": "width: 1024, height: 1024, sampler: DPM++ 2M EDM"
+    "extras": "sampler: DPM++ 2M EDM"
+  },
+
+  "CogView 3 Plus": {
+    "path": "THUDM/CogView3-Plus-3B",
+    "desc": "This model is the DiT version of CogView3, a text-to-image generation model, supporting image generation from 512 to 2048px. Resolution: Width and height must meet the range from 512px to 2048px and must be divisible by 32.",
+    "preview": "THUDM--CogView3-Plus-3B.jpg",
+    "skip": true
+  },
+  "Meissonic": {
+    "path": "MeissonFlow/Meissonic",
+    "desc": "Meissonic is a non-autoregressive mask image modeling text-to-image synthesis model that can generate high-resolution images. It is designed to run on consumer graphics cards.",
+    "preview": "MeissonFlow--Meissonic.jpg",
+    "skip": true
  },

  "aMUSEd 256": {
@@ -326,7 +360,7 @@
    "path": "warp-ai/wuerstchen",
    "desc": "Würstchen is a diffusion model whose text-conditional model works in a highly compressed latent space of images. Why is this important? Compressing data can reduce computational costs for both training and inference by magnitudes. Training on 1024x1024 images, is way more expensive than training at 32x32. Usually, other works make use of a relatively small compression, in the range of 4x - 8x spatial compression. Würstchen takes this to an extreme. Through its novel design, we achieve a 42x spatial compression. Würstchen employs a two-stage compression, what we call Stage A and Stage B. Stage A is a VQGAN, and Stage B is a Diffusion Autoencoder (more details can be found in the paper). A third model, Stage C, is learned in that highly compressed latent space. This training requires fractions of the compute used for current top-performing models, allowing also cheaper and faster inference.",
    "preview": "warp-ai--wuerstchen.jpg",
-    "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 4.0, image_cfg_scale: 0.0"
+    "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 0.0"
  },
  "KOALA 700M": {
    "path": "huggingface/etri-vilab/koala-700m-llava-cap",
@@ -334,7 +368,7 @@
    "skip": true,
    "desc": "Fast text-to-image model, called KOALA, by compressing SDXL's U-Net and distilling knowledge from SDXL into our model. KOALA-700M can generate a 1024x1024 image in less than 1.5 seconds on an NVIDIA 4090 GPU, which is more than 2x faster than SDXL.",
    "preview": "etri-vilab--koala-700m-llava-cap.jpg",
-    "extras": "width: 1024, height: 1024, sampler: Default"
+    "extras": "sampler: Default"
  },
  "Tsinghua UniDiffuser": {
    "path": "thu-ml/unidiffuser-v1",
@@ -356,7 +390,13 @@
    "path": "DeepFloyd/IF-I-M-v1.0",
    "desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.",
    "preview": "DeepFloyd--IF-I-M-v1.0.jpg",
-    "extras": "width: 1024, height: 1024, sampler: Default"
+    "extras": "sampler: Default"
+  },
+  "DeepFloyd IF Large": {
+    "path": "DeepFloyd/IF-I-L-v1.0",
+    "desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.",
+    "preview": "DeepFloyd--IF-I-M-v1.0.jpg",
+    "extras": "sampler: Default"
  }

 }
@@ -9,12 +9,6 @@ import platform
 import subprocess
 import cProfile

-try:
-    import pkg_resources # python 3.12 no longer has it built-in
-except ImportError:
-    stdout = subprocess.run(f'"{sys.executable}" -m pip install setuptools', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    import pkg_resources
-

 class Dot(dict): # dot notation access to dictionary attributes
    __getattr__ = dict.get
@@ -22,6 +16,7 @@ class Dot(dict): # dot notation access to dictionary attributes
    __delattr__ = dict.__delitem__


+pkg_resources, setuptools, distutils = None, None, None # defined via ensure_base_requirements
 version = None
 current_branch = None
 log = logging.getLogger("sd")
@@ -57,8 +52,9 @@ args = Dot({
 })
 git_commit = "unknown"
 diffusers_commit = "unknown"
-submodules_commit = {
+extensions_commit = {
    'sd-webui-controlnet': 'ecd33eb',
+    'adetailer': 'a89c01d'
    # 'stable-diffusion-webui-images-browser': '27fe4a7',
 }

@@ -88,15 +84,12 @@ def setup_logging():
        def get(self):
            return self.buffer

-    install('rich', 'rich', quiet=True)
-    install('setuptools==69.5.1', 'setuptools', quiet=True)
-    install('psutil', 'psutil', quiet=True)
-    install('requests', 'requests', quiet=True)
    from functools import partial, partialmethod
    from logging.handlers import RotatingFileHandler
    from rich.theme import Theme
    from rich.logging import RichHandler
    from rich.console import Console
+    from rich import print as rprint
    from rich.pretty import install as pretty_install
    from rich.traceback import install as traceback_install

@@ -111,6 +104,7 @@ def setup_logging():

    level = logging.DEBUG if args.debug else logging.INFO
    log.setLevel(logging.DEBUG) # log to file is always at level debug for facility `sd`
+    log.print = rprint
    global console # pylint: disable=global-statement
    console = Console(log_time=True, log_time_format='%H:%M:%S-%f', theme=Theme({
        "traceback.border": "black",
@@ -119,7 +113,7 @@ def setup_logging():
    }))
    logging.basicConfig(level=logging.ERROR, format='%(asctime)s | %(name)s | %(levelname)s | %(module)s | %(message)s', handlers=[logging.NullHandler()]) # redirect default logger to null
    pretty_install(console=console)
-    traceback_install(console=console, extra_lines=1, max_frames=10, width=console.width, word_wrap=False, indent_guides=False, suppress=[])
+    traceback_install(console=console, extra_lines=1, max_frames=16, width=console.width, word_wrap=False, indent_guides=False, suppress=[])
    while log.hasHandlers() and len(log.handlers) > 0:
        log.removeHandler(log.handlers[0])

@@ -172,14 +166,35 @@ def custom_excepthook(exc_type, exc_value, exc_traceback):


 def print_dict(d):
+    if d is None:
+        return ''
    return ' '.join([f'{k}={v}' for k, v in d.items()])


 def print_profile(profiler: cProfile.Profile, msg: str):
+    profiler.disable()
    from modules.errors import profile
    profile(profiler, msg)


+@lru_cache()
+def package_version(package):
+    try:
+        return pkg_resources.get_distribution(package).version
+    except Exception:
+        return None
+
+
+@lru_cache()
+def package_spec(package):
+    spec = pkg_resources.working_set.by_key.get(package, None) # more reliable than importlib
+    if spec is None:
+        spec = pkg_resources.working_set.by_key.get(package.lower(), None) # check name variations
+    if spec is None:
+        spec = pkg_resources.working_set.by_key.get(package.replace('_', '-'), None) # check name variations
+    return spec
+
+
 # check if package is installed
@lru_cache()
 def installed(package, friendly: str = None, reload = False, quiet = False):
@@ -187,8 +202,8 @@ def installed(package, friendly: str = None, reload = False, quiet = False):
    try:
        if reload:
            try:
-                import imp # pylint: disable=deprecated-module
-                imp.reload(pkg_resources)
+                import importlib # pylint: disable=deprecated-module
+                importlib.reload(pkg_resources)
            except Exception:
                pass
        if friendly:
@@ -197,33 +212,31 @@ def installed(package, friendly: str = None, reload = False, quiet = False):
            pkgs = [p for p in package.split() if not p.startswith('-') and not p.startswith('=')]
            pkgs = [p.split('/')[-1] for p in pkgs] # get only package name if installing from url
        for pkg in pkgs:
-            if '>=' in pkg:
+            if '!=' in pkg:
+                p = pkg.split('!=')
+                return True # check for not equal always return true
+            elif '>=' in pkg:
                p = pkg.split('>=')
            else:
                p = pkg.split('==')
-            spec = pkg_resources.working_set.by_key.get(p[0], None) # more reliable than importlib
-            if spec is None:
-                spec = pkg_resources.working_set.by_key.get(p[0].lower(), None) # check name variations
-            if spec is None:
-                spec = pkg_resources.working_set.by_key.get(p[0].replace('_', '-'), None) # check name variations
+            spec = package_spec(p[0])
            ok = ok and spec is not None
            if ok:
-                package_version = pkg_resources.get_distribution(p[0]).version
-                # log.debug(f"Package version found: {p[0]} {package_version}")
+                pkg_version = package_version(p[0])
                if len(p) > 1:
-                    exact = package_version == p[1]
+                    exact = pkg_version == p[1]
                    if not exact and not quiet:
                        if args.experimental:
-                            log.warning(f"Package allowing experimental: {p[0]} {package_version} required {p[1]}")
+                            log.warning(f"Package: {p[0]} {pkg_version} required {p[1]} allowing experimental")
                        else:
-                            log.warning(f"Package version mismatch: {p[0]} {package_version} required {p[1]}")
+                            log.warning(f"Package: {p[0]} {pkg_version} required {p[1]} version mismatch")
                    ok = ok and (exact or args.experimental)
            else:
                if not quiet:
-                    log.debug(f"Package not found: {p[0]}")
+                    log.debug(f"Package: {p[0]} not found")
        return ok
    except Exception as e:
-        log.debug(f"Package error: {pkgs} {e}")
+        log.error(f"Package: {pkgs} {e}")
        return False


@@ -233,7 +246,7 @@ def uninstall(package, quiet = False):
    for p in packages:
        if installed(p, p, quiet=True):
            if not quiet:
-                log.warning(f'Uninstalling: {p}')
+                log.warning(f'Package: {p} uninstall')
            res += pip(f"uninstall {p} --yes --quiet", ignore=True, quiet=True)
    return res

@@ -248,12 +261,15 @@ def pip(arg: str, ignore: bool = False, quiet: bool = False, uv = True):
        log.info(f'Install: package="{arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip()}" mode={"uv" if uv else "pip"}')
    env_args = os.environ.get("PIP_EXTRA_ARGS", "")
    all_args = f'{pip_log}{arg} {env_args}'.strip()
-    log.debug(f'Running: {pipCmd}="{all_args}"')
+    if not quiet:
+        log.debug(f'Running: {pipCmd}="{all_args}"')
    result = subprocess.run(f'"{sys.executable}" -m {pipCmd} {all_args}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    txt = result.stdout.decode(encoding="utf8", errors="ignore")
    if len(result.stderr) > 0:
        if uv and result.returncode != 0:
-            log.warning('Cannot install with uv, fallback to pip')
+            err = result.stderr.decode(encoding="utf8", errors="ignore")
+            log.warning('Install: cannot use uv, fallback to pip')
+            debug(f'Install: uv pip error: {err}')
            return pip(originalArg, ignore, quiet, uv=False)
        else:
            txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore")
@@ -262,8 +278,8 @@ def pip(arg: str, ignore: bool = False, quiet: bool = False, uv = True):
    if result.returncode != 0 and not ignore:
        global errors # pylint: disable=global-statement
        errors += 1
-        log.error(f'Error running {pipCmd}: {arg}')
-        log.debug(f'Pip output: {txt}')
+        log.error(f'Install: {pipCmd}: {arg}')
+        log.debug(f'Install: pip output {txt}')
    return txt


@@ -278,8 +294,8 @@ def install(package, friendly: str = None, ignore: bool = False, reinstall: bool
        deps = '' if not no_deps else '--no-deps '
        res = pip(f"install{' --upgrade' if not args.uv else ''} {deps}{package}", ignore=ignore, uv=package != "uv")
        try:
-            import imp # pylint: disable=deprecated-module
-            imp.reload(pkg_resources)
+            import importlib # pylint: disable=deprecated-module
+            importlib.reload(pkg_resources)
        except Exception:
            pass
    return res
@@ -306,9 +322,9 @@ def git(arg: str, folder: str = None, ignore: bool = False, optional: bool = Fal
            return txt
        global errors # pylint: disable=global-statement
        errors += 1
-        log.error(f'Error running git: {folder} / {arg}')
+        log.error(f'Git: {folder} / {arg}')
        if 'or stash them' in txt:
-            log.error(f'Local changes detected: check log for details: {log_file}')
+            log.error(f'Git local changes detected: check details log="{log_file}"')
        log.debug(f'Git output: {txt}')
    return txt

@@ -337,7 +353,7 @@ def branch(folder=None):
        b = 'master'
    else:
        b = b.split('\n')[0].replace('*', '').strip()
-    log.debug(f'Submodule: {folder} / {b}')
+    log.debug(f'Git submodule: {folder} / {b}')
    git(f'checkout {b}', folder, ignore=True, optional=True)
    return b

@@ -360,7 +376,7 @@ def update(folder, keep_branch = False, rebase = True):
    else:
        res = git(f'pull origin {b} {arg}', folder)
        debug(f'Install update: folder={folder} branch={b} args={arg} {res}')
-    commit = submodules_commit.get(os.path.basename(folder), None)
+    commit = extensions_commit.get(os.path.basename(folder), None)
    if commit is not None:
        res = git(f'checkout {commit}', folder)
        debug(f'Install update: folder={folder} branch={b} args={arg} commit={commit} {res}')
@@ -410,15 +426,15 @@ def get_platform():
 def check_python(supported_minors=[9, 10, 11, 12], reason=None):
    if args.quick:
        return
-    log.info(f'Python version={platform.python_version()} platform={platform.system()} bin="{sys.executable}" venv="{sys.prefix}"')
+    log.info(f'Python: version={platform.python_version()} platform={platform.system()} bin="{sys.executable}" venv="{sys.prefix}"')
    if int(sys.version_info.major) == 3 and int(sys.version_info.minor) == 12 and int(sys.version_info.micro) > 3: # TODO python 3.12.4 or higher cause a mess with pydantic
-        log.error(f"Incompatible Python version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro} required 3.12.3 or lower")
+        log.error(f"Python version incompatible: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro} required 3.12.3 or lower")
        if reason is not None:
            log.error(reason)
        if not args.ignore:
            sys.exit(1)
    if not (int(sys.version_info.major) == 3 and int(sys.version_info.minor) in supported_minors):
-        log.error(f"Incompatible Python version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro} required 3.{supported_minors}")
+        log.error(f"Python version incompatible: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro} required 3.{supported_minors}")
        if reason is not None:
            log.error(reason)
        if not args.ignore:
@@ -433,17 +449,17 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None):
                sys.exit(1)
    else:
        git_version = git('--version', folder=None, ignore=False)
-        log.debug(f'Git {git_version.replace("git version", "").strip()}')
+        log.debug(f'Git: version={git_version.replace("git version", "").strip()}')


 # check diffusers version
 def check_diffusers():
-    sha = '5e1427a7da6e878b958fd5a2422c7763a94ff02b'
+    sha = 'e45c25d03aeb0a967d8aaa0f6a79f280f6838e1f'
    pkg = pkg_resources.working_set.by_key.get('diffusers', None)
    minor = int(pkg.version.split('.')[1] if pkg is not None else 0)
    cur = opts.get('diffusers_version', '') if minor > 0 else ''
    if (minor == 0) or (cur != sha):
-        log.debug(f'Diffusers {"install" if minor == 0 else "upgrade"}: current={pkg}@{cur} target={sha}')
+        log.debug(f'Diffusers {"install" if minor == 0 else "upgrade"}: package={pkg} current={cur} target={sha}')
        if minor > 0:
            pip('uninstall --yes diffusers', ignore=True, quiet=True, uv=False)
        pip(f'install --upgrade git+https://github.com/huggingface/diffusers@{sha}', ignore=False, quiet=True, uv=False)
@@ -459,50 +475,77 @@ def check_onnx():
        install('onnxruntime', 'onnxruntime', ignore=True)


+def check_torchao():
+    if installed('torchao', quiet=True):
+        ver = package_version('torchao')
+        if ver != '0.5.0':
+            log.debug(f'Uninstall: torchao=={ver}')
+            pip('uninstall --yes torchao', ignore=True, quiet=True, uv=False)
+            for m in [m for m in sys.modules if m.startswith('torchao')]:
+                del sys.modules[m]
+
+
 def install_cuda():
-    log.info('nVidia CUDA toolkit detected: nvidia-smi present')
+    log.info('CUDA: nVidia toolkit detected')
    install('onnxruntime-gpu', 'onnxruntime-gpu', ignore=True, quiet=True)
-    return os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/cu124')
+    # return os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/cu124')
+    return os.environ.get('TORCH_COMMAND', 'torch==2.4.1+cu124 torchvision==0.19.1+cu124 --index-url https://download.pytorch.org/whl/cu124')


 def install_rocm_zluda():
    from modules import rocm

    if not rocm.is_installed:
-        log.warning('Could not find ROCm toolkit installed.')
+        log.warning('ROCm: could not find ROCm toolkit installed')
        log.info('Using CPU-only torch')
        return os.environ.get('TORCH_COMMAND', 'torch torchvision')

    check_python(supported_minors=[10, 11], reason='ROCm or ZLUDA backends require Python 3.10 or 3.11')
-    log.info('AMD ROCm toolkit detected')
+    log.info('ROCm: AMD toolkit detected')
    os.environ.setdefault('PYTORCH_HIP_ALLOC_CONF', 'garbage_collection_threshold:0.8,max_split_size_mb:512')
    # if not is_windows:
    #    os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow-rocm')
+
+    device = None
    try:
        amd_gpus = rocm.get_agents()
-        log.info(f'ROCm agents detected: {[gpu.name for gpu in amd_gpus]}')
+        if len(amd_gpus) == 0:
+            log.warning('ROCm: no agent was found')
+        else:
+            log.info(f'ROCm: agents={[gpu.name for gpu in amd_gpus]}')
+            if args.device_id is None:
+                index = 0
+                for idx, gpu in enumerate(amd_gpus):
+                    index = idx
+                    # if gpu.name.startswith('gfx11') and os.environ.get('TENSORFLOW_PACKAGE') == 'tensorflow-rocm': # do not use tensorflow-rocm for navi 3x
+                    #    os.environ['TENSORFLOW_PACKAGE'] = 'tensorflow==2.13.0'
+                    if not gpu.is_apu:
+                        # although apu was found, there can be a dedicated card. do not break loop.
+                        # if no dedicated card was found, apu will be used.
+                        break
+                os.environ.setdefault('HIP_VISIBLE_DEVICES', str(index))
+                device = amd_gpus[index]
+            else:
+                device_id = int(args.device_id)
+                if device_id < len(amd_gpus):
+                    device = amd_gpus[device_id]
    except Exception as e:
        log.warning(f'ROCm agent enumerator failed: {e}')
-        amd_gpus = []

-    hip_default_device = None
-    for idx, gpu in enumerate(amd_gpus):
-        gfx_version = gpu.get_gfx_version()
-        if gfx_version is None:
-            log.debug(f'HSA_OVERRIDE_GFX_VERSION auto config is skipped for {gpu.name}')
-        else:
-            hip_default_device = gpu
-            log.debug(f'ROCm agent used by default: idx={idx} gpu={gpu.name}')
-            os.environ.setdefault('HIP_VISIBLE_DEVICES', str(idx))
-            # if os.environ.get('TENSORFLOW_PACKAGE') == 'tensorflow-rocm': # do not use tensorflow-rocm for navi 3x
-            #    os.environ['TENSORFLOW_PACKAGE'] = 'tensorflow==2.13.0'
-            os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', gfx_version)
-            break
-
-    log.info(f'ROCm version detected: {rocm.version}')
+    msg = f'ROCm: version={rocm.version}'
+    if device is not None:
+        msg += f', using agent {device.name}'
+    log.info(msg)
    torch_command = ''
    if sys.platform == "win32":
-        #if args.use_zluda:
+        # TODO after ROCm for Windows is released
+
+        if args.device_id is not None:
+            if os.environ.get('HIP_VISIBLE_DEVICES', None) is not None:
+                log.warning('Setting HIP_VISIBLE_DEVICES and --device-id at the same time may be mistake.')
+            os.environ['HIP_VISIBLE_DEVICES'] = args.device_id
+            del args.device_id
+
        log.warning("ZLUDA support: experimental")
        error = None
        from modules import zluda_installer
@@ -516,12 +559,9 @@ def install_rocm_zluda():
            error = e
            log.warning(f'Failed to install ZLUDA: {e}')
        if error is None:
-            if args.device_id is not None:
-                os.environ['HIP_VISIBLE_DEVICES'] = args.device_id
-                del args.device_id
            try:
                zluda_installer.load(zluda_path)
-                torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.0 torchvision --index-url https://download.pytorch.org/whl/cu118')
+                torch_command = os.environ.get('TORCH_COMMAND', f'torch=={zluda_installer.get_default_torch_version(device)} torchvision --index-url https://download.pytorch.org/whl/cu118')
                log.info(f'Using ZLUDA in {zluda_path}')
            except Exception as e:
                error = e
@@ -529,14 +569,15 @@ def install_rocm_zluda():
        if error is not None:
            log.info('Using CPU-only torch')
            torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision')
-        #else:
-        # TODO after ROCm for Windows is released
    else:
-        if rocm.version is None or float(rocm.version) > 6.1: # assume the latest if version check fails
-            torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/rocm6.1')
+        if rocm.version is None or float(rocm.version) >= 6.1: # assume the latest if version check fails
+            #torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/rocm6.1')
+            torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.4.1+rocm6.1 torchvision==0.19.1+rocm6.1 --index-url https://download.pytorch.org/whl/rocm6.1')
+        elif rocm.version == "6.0": # lock to 2.4.1, older rocm (5.7) uses torch 2.3
+            torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.4.1+rocm6.0 torchvision==0.19.1+rocm6.0 --index-url https://download.pytorch.org/whl/rocm6.0')
        elif float(rocm.version) < 5.5: # oldest supported version is 5.5
-            log.warning(f"Unsupported ROCm version detected: {rocm.version}")
-            log.warning("Minimum supported ROCm version is 5.5")
+            log.warning(f"ROCm: unsupported version={rocm.version}")
+            log.warning("ROCm: minimum supported version=5.5")
            torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/rocm5.5')
        else:
            torch_command = os.environ.get('TORCH_COMMAND', f'torch torchvision --index-url https://download.pytorch.org/whl/rocm{rocm.version}')
@@ -549,42 +590,43 @@ def install_rocm_zluda():
                ort_package = os.environ.get('ONNXRUNTIME_PACKAGE', f"--pre onnxruntime-training{'' if ort_version is None else ('==' + ort_version)} --index-url https://pypi.lsh.sh/{rocm.version[0]}{rocm.version[2]} --extra-index-url https://pypi.org/simple")
            install(ort_package, 'onnxruntime-training')

-        if hip_default_device is not None and rocm.version != "6.2" and rocm.version == rocm.version_torch and rocm.get_blaslt_enabled():
-            log.debug(f'hipBLASLt arch={hip_default_device.name} available={hip_default_device.blaslt_supported}')
-            rocm.set_blaslt_enabled(hip_default_device.blaslt_supported)
+        if installed("torch") and device is not None:
+            if 'Flash attention' in opts.get('sdp_options'):
+                if not installed('flash-attn'):
+                    install(rocm.get_flash_attention_command(device), reinstall=True)
+            elif not args.experimental:
+                uninstall('flash-attn')
+
+        if device is not None and rocm.version != "6.2" and rocm.version == rocm.version_torch and rocm.get_blaslt_enabled():
+            log.debug(f'ROCm hipBLASLt: arch={device.name} available={device.blaslt_supported}')
+            rocm.set_blaslt_enabled(device.blaslt_supported)
+
+    if device is None:
+        log.debug('ROCm: HSA_OVERRIDE_GFX_VERSION auto config skipped')
+    else:
+        gfx_ver = device.get_gfx_version()
+        if gfx_ver is not None:
+            os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', gfx_ver)
+        else:
+            log.warning(f'ROCm: device={device.name} could not auto-detect HSA version')
+
    return torch_command


 def install_ipex(torch_command):
    check_python(supported_minors=[10,11], reason='IPEX backend requires Python 3.10 or 3.11')
    args.use_ipex = True # pylint: disable=attribute-defined-outside-init
-    log.info('Intel OneAPI Toolkit detected')
+    log.info('IPEX: Intel OneAPI toolkit detected')
    if os.environ.get("NEOReadDebugKeys", None) is None:
        os.environ.setdefault('NEOReadDebugKeys', '1')
    if os.environ.get("ClDeviceGlobalMemSizeAvailablePercent", None) is None:
        os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100')
    if "linux" in sys.platform:
        torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.1+cxx11.abi torchvision==0.18.1+cxx11.abi intel-extension-for-pytorch==2.3.110+xpu oneccl_bind_pt==2.3.100+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/')
+        # torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/test/xpu') # test wheels are stable previews, significantly slower than IPEX
        # os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow==2.15.1 intel-extension-for-tensorflow[xpu]==2.15.0.1')
    else:
-        if sys.version_info.minor == 11:
-            pytorch_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/torch-2.1.0a0+cxx11.abi-cp311-cp311-win_amd64.whl'
-            torchvision_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/torchvision-0.16.0a0+cxx11.abi-cp311-cp311-win_amd64.whl'
-            ipex_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/intel_extension_for_pytorch-2.1.10+xpu-cp311-cp311-win_amd64.whl'
-            torch_command = os.environ.get('TORCH_COMMAND', f'{pytorch_pip} {torchvision_pip} {ipex_pip}')
-        elif sys.version_info.minor == 10:
-            pytorch_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/torch-2.1.0a0+cxx11.abi-cp310-cp310-win_amd64.whl'
-            torchvision_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/torchvision-0.16.0a0+cxx11.abi-cp310-cp310-win_amd64.whl'
-            ipex_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/intel_extension_for_pytorch-2.1.10+xpu-cp310-cp310-win_amd64.whl'
-            torch_command = os.environ.get('TORCH_COMMAND', f'{pytorch_pip} {torchvision_pip} {ipex_pip}')
-        else:
-            torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.1.0.post3 torchvision==0.16.0.post3 intel-extension-for-pytorch==2.1.40+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/')
-            if os.environ.get('DISABLE_VENV_LIBS', None) is None:
-                install(os.environ.get('MKL_PACKAGE', 'mkl==2024.2.0'), 'mkl')
-                install(os.environ.get('DPCPP_PACKAGE', 'mkl-dpcpp==2024.2.0'), 'mkl-dpcpp')
-                install(os.environ.get('ONECCL_PACKAGE', 'oneccl-devel==2021.13.0'), 'oneccl-devel')
-                install(os.environ.get('MPI_PACKAGE', 'impi-devel==2021.13.0'), 'impi-devel')
-        torch_command = os.environ.get('TORCH_COMMAND', f'{pytorch_pip} {torchvision_pip} {ipex_pip}')
+        torch_command = os.environ.get('TORCH_COMMAND', '--pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/xpu') # torchvision doesn't exist on test/stable branch for windows
    install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2024.3.0'), 'openvino', ignore=True)
    install('nncf==2.7.0', 'nncf', ignore=True)
    install(os.environ.get('ONNXRUNTIME_PACKAGE', 'onnxruntime-openvino'), 'onnxruntime-openvino', ignore=True)
@@ -593,8 +635,8 @@ def install_ipex(torch_command):

 def install_openvino(torch_command):
    check_python(supported_minors=[8, 9, 10, 11, 12], reason='OpenVINO backend requires Python 3.9, 3.10 or 3.11')
-    log.info('Using OpenVINO')
-    torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.1 torchvision==0.18.1 --index-url https://download.pytorch.org/whl/cpu')
+    log.info('OpenVINO: selected')
+    torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.1+cpu torchvision==0.18.1+cpu --index-url https://download.pytorch.org/whl/cpu')
    install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2024.3.0'), 'openvino')
    install(os.environ.get('ONNXRUNTIME_PACKAGE', 'onnxruntime-openvino'), 'onnxruntime-openvino', ignore=True)
    install('nncf==2.12.0', 'nncf')
@@ -615,7 +657,7 @@ def install_torch_addons():
            import torch # pylint: disable=unused-import
            import xformers # pylint: disable=unused-import
        except Exception as e:
-            log.debug(f'Cannot install xformers package: {e}')
+            log.debug(f'xFormers cannot install: {e}')
    elif not args.experimental and not args.use_xformers and opts.get('cross_attention_optimization', '') != 'xFormers':
        uninstall('xformers')
    if opts.get('cuda_compile_backend', '') == 'hidet':
@@ -635,7 +677,7 @@ def install_torch_addons():
 # check torch version
 def check_torch():
    if args.skip_torch:
-        log.info('Skipping Torch tests')
+        log.info('Torch: skip tests')
        return
    if args.profile:
        pr = cProfile.Profile()
@@ -646,8 +688,8 @@ def check_torch():
    allow_ipex = not (args.use_cuda or args.use_rocm or args.use_directml or args.use_openvino)
    allow_directml = not (args.use_cuda or args.use_rocm or args.use_ipex or args.use_openvino)
    allow_openvino = not (args.use_cuda or args.use_rocm or args.use_ipex or args.use_directml)
-    log.debug(f'Torch overrides: cuda={args.use_cuda} rocm={args.use_rocm} ipex={args.use_ipex} diml={args.use_directml} openvino={args.use_openvino}')
-    log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml} openvino={allow_openvino}')
+    log.debug(f'Torch overrides: cuda={args.use_cuda} rocm={args.use_rocm} ipex={args.use_ipex} directml={args.use_directml} openvino={args.use_openvino} zluda={args.use_zluda}')
+    # log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml} openvino={allow_openvino}')
    torch_command = os.environ.get('TORCH_COMMAND', '')

    if torch_command != '':
@@ -680,8 +722,8 @@ def check_torch():
            if sys.platform == 'darwin':
                torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision')
            elif allow_directml and args.use_directml and ('arm' not in machine and 'aarch' not in machine):
-                log.info('Using DirectML Backend')
-                torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.1 torchvision torch-directml')
+                log.info('DirectML: selected')
+                torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.4.1 torchvision torch-directml')
                if 'torch' in torch_command and not args.version:
                    install(torch_command, 'torch torchvision')
                install('onnxruntime-directml', 'onnxruntime-directml', ignore=True)
@@ -701,7 +743,7 @@ def check_torch():
                    import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
                    log.info(f'Torch backend: Intel IPEX {ipex.__version__}')
                except Exception:
-                    log.warning('IPEX not found')
+                    log.warning('IPEX: not found')
                if shutil.which('icpx') is not None:
                    log.info(f'{os.popen("icpx --version").read().rstrip()}')
                for device in range(torch.xpu.device_count()):
@@ -727,7 +769,7 @@ def check_torch():
                except Exception:
                    log.warning("Torch reports CUDA not available")
        except Exception as e:
-            log.error(f'Could not load torch: {e}')
+            log.error(f'Torch cannot load: {e}')
            if not args.ignore:
                sys.exit(1)
    if rocm.is_installed:
@@ -737,12 +779,13 @@ def check_torch():
            try:
                rocm.load_hsa_runtime()
            except OSError:
-                log.error("Failed to preload HSA Runtime library.")
+                log.error("ROCm: failed to preload HSA runtime")
    if args.version:
        return
    if not args.skip_all:
        install_torch_addons()
    if args.profile:
+        pr.disable()
        print_profile(pr, 'Torch')


@@ -778,12 +821,8 @@ def install_packages():
    # tensorflow_package = os.environ.get('TENSORFLOW_PACKAGE', None)
    # if tensorflow_package is not None:
    #    install(tensorflow_package, 'tensorflow-rocm' if 'rocm' in tensorflow_package else 'tensorflow', ignore=True, quiet=True)
-    # bitsandbytes_package = os.environ.get('BITSANDBYTES_PACKAGE', None)
-    # if bitsandbytes_package is not None:
-    #    install(bitsandbytes_package, 'bitsandbytes', ignore=True, quiet=True)
-    # elif not args.experimental:
-    #    uninstall('bitsandbytes')
    if args.profile:
+        pr.disable( )
        print_profile(pr, 'Packages')


@@ -793,21 +832,21 @@ def run_extension_installer(folder):
    if not os.path.isfile(path_installer):
        return
    try:
-        log.debug(f"Running extension installer: {path_installer}")
+        log.debug(f"Extension installer: {path_installer}")
        env = os.environ.copy()
        env['PYTHONPATH'] = os.path.abspath(".")
        result = subprocess.run(f'"{sys.executable}" "{path_installer}"', shell=True, env=env, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=folder)
        txt = result.stdout.decode(encoding="utf8", errors="ignore")
-        debug(f'Extension installer: file={path_installer} {txt}')
+        debug(f'Extension installer: file="{path_installer}" {txt}')
        if result.returncode != 0:
            global errors # pylint: disable=global-statement
            errors += 1
            if len(result.stderr) > 0:
                txt = txt + '\n' + result.stderr.decode(encoding="utf8", errors="ignore")
-            log.error(f'Error running extension installer: {path_installer}')
+            log.error(f'Extension installer error: {path_installer}')
            log.debug(txt)
    except Exception as e:
-        log.error(f'Exception running extension installer: {e}')
+        log.error(f'Extension installer exception: {e}')

 # get list of all enabled extensions
 def list_extensions_folder(folder, quiet=False):
@@ -849,9 +888,13 @@ def install_extensions(force=False):
                try:
                    res.append(update(os.path.join(folder, ext)))
                except Exception:
-                    res.append(f'Error updating extension: {os.path.join(folder, ext)}')
-                    log.error(f'Error updating extension: {os.path.join(folder, ext)}')
+                    res.append(f'Extension update error: {os.path.join(folder, ext)}')
+                    log.error(f'Extension update error: {os.path.join(folder, ext)}')
            if not args.skip_extensions:
+                commit = extensions_commit.get(os.path.basename(ext), None)
+                if commit is not None:
+                    log.debug(f'Extension force: name="{ext}" commit={commit}')
+                    res.append(git(f'checkout {commit}', os.path.join(folder, ext)))
                run_extension_installer(os.path.join(folder, ext))
            pkg_resources._initialize_master_working_set() # pylint: disable=protected-access
            try:
@@ -866,6 +909,7 @@ def install_extensions(force=False):
    if len(extensions_duplicates) > 0:
        log.warning(f'Extensions duplicates: {extensions_duplicates}')
    if args.profile:
+        pr.disable()
        print_profile(pr, 'Extensions')
    return '\n'.join(res)

@@ -894,30 +938,47 @@ def install_submodules(force=True):
            else:
                branch(name)
        except Exception:
-            log.error(f'Error updating submodule: {submodule}')
+            log.error(f'Submodule update error: {submodule}')
    setup_logging()
    if args.profile:
+        pr.disable()
        print_profile(pr, 'Submodule')
    return '\n'.join(res)


 def ensure_base_requirements():
+    setuptools_version = '69.5.1'
+
+    def update_setuptools():
+        # print('Install base requirements')
+        global pkg_resources, setuptools, distutils # pylint: disable=global-statement
+        # python may ship with incompatible setuptools
+        subprocess.run(f'"{sys.executable}" -m pip install setuptools=={setuptools_version}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        import importlib
+        # need to delete all references to modules to be able to reload them otherwise python will use cached version
+        modules = [m for m in sys.modules if m.startswith('setuptools') or m.startswith('pkg_resources') or m.startswith('distutils')]
+        for m in modules:
+            del sys.modules[m]
+        setuptools = importlib.import_module('setuptools')
+        sys.modules['setuptools'] = setuptools
+        distutils = importlib.import_module('distutils')
+        sys.modules['distutils'] = distutils
+        pkg_resources = importlib.import_module('pkg_resources')
+        sys.modules['pkg_resources'] = pkg_resources
+
    try:
-        import setuptools # pylint: disable=unused-import
+        global pkg_resources, setuptools # pylint: disable=global-statement
+        import pkg_resources # pylint: disable=redefined-outer-name
+        import setuptools # pylint: disable=redefined-outer-name
+        if setuptools.__version__ != setuptools_version:
+            update_setuptools()
    except ImportError:
-        install('setuptools==69.5.1', 'setuptools')
-    try:
-        import setuptools # pylint: disable=unused-import
-    except ImportError:
-        pass
-    try:
-        import rich # pylint: disable=unused-import
-    except ImportError:
-        install('rich', 'rich')
-    try:
-        import rich # pylint: disable=unused-import
-    except ImportError:
-        pass
+        update_setuptools()
+
+    # used by installler itself so must be installed before requirements
+    install('rich', 'rich', quiet=True)
+    install('psutil', 'psutil', quiet=True)
+    install('requests', 'requests', quiet=True)


 def install_requirements():
@@ -939,6 +1000,7 @@ def install_requirements():
            if not installed(line, quiet=True):
                _res = install(line)
    if args.profile:
+        pr.disable()
        print_profile(pr, 'Requirements')


@@ -967,11 +1029,14 @@ def set_environment():
    os.environ.setdefault('UVICORN_TIMEOUT_KEEP_ALIVE', '60')
    os.environ.setdefault('KINETO_LOG_LEVEL', '3')
    os.environ.setdefault('DO_NOT_TRACK', '1')
+    os.environ.setdefault('UV_INDEX_STRATEGY', 'unsafe-any-match')
+    os.environ.setdefault('UV_NO_BUILD_ISOLATION', '1')
    os.environ.setdefault('HF_HUB_CACHE', opts.get('hfcache_dir', os.path.join(os.path.expanduser('~'), '.cache', 'huggingface', 'hub')))
-    log.info(f'HF cache folder: {os.environ.get("HF_HUB_CACHE")}')
    allocator = f'garbage_collection_threshold:{opts.get("torch_gc_threshold", 80)/100:0.2f},max_split_size_mb:512'
    if opts.get("torch_malloc", "native") == 'cudaMallocAsync':
        allocator += ',backend:cudaMallocAsync'
+    if opts.get("torch_expandable_segments", False):
+        allocator += ',expandable_segments:True'
    os.environ.setdefault('PYTORCH_CUDA_ALLOC_CONF', allocator)
    log.debug(f'Torch allocator: "{allocator}"')
    if sys.platform == 'darwin':
@@ -1104,18 +1169,18 @@ def check_version(offline=False, reset=True): # pylint: disable=unused-argument
                    update('.', keep_branch=True)
                    # git('git stash pop')
                    ver = git('log -1 --pretty=format:"%h %ad"')
-                    log.info(f'Upgraded to version: {ver}')
+                    log.info(f'Repository upgraded: {ver}')
                except Exception:
                    if not reset:
-                        log.error('Error during repository upgrade')
+                        log.error('Repository error upgrading')
                    else:
-                        log.warning('Retrying repository upgrade...')
+                        log.warning('Repository: retrying upgrade...')
                        git_reset()
                        check_version(offline=offline, reset=False)
            else:
-                log.info(f'Latest published version: {commits["commit"]["sha"]} {commits["commit"]["commit"]["author"]["date"]}')
+                log.info(f'Repository latest available {commits["commit"]["sha"]} {commits["commit"]["commit"]["author"]["date"]}')
    except Exception as e:
-        log.error(f'Failed to check version: {e} {commits}')
+        log.error(f'Repository failed to check version: {e} {commits}')


 def update_wiki():
@@ -1124,7 +1189,7 @@ def update_wiki():
        try:
            update(os.path.join(os.path.dirname(__file__), "wiki"))
        except Exception:
-            log.error('Error updating wiki')
+            log.error('Wiki update error')


 # check if we can run setup in quick mode
@@ -1146,18 +1211,18 @@ def check_timestamp():
    try:
        version_time = int(git('log -1 --pretty=format:"%at"'))
    except Exception as e:
-        log.error(f'Error getting local repository version: {e}')
-    log.debug(f'Repository update time: {time.ctime(int(version_time))}')
+        log.error(f'Timestamp local repository version: {e}')
+    log.debug(f'Timestamp repository update time: {time.ctime(int(version_time))}')
    if setup_time == -1:
        return False
-    log.debug(f'Previous setup time: {time.ctime(setup_time)}')
+    log.debug(f'Timestamp previous setup time: {time.ctime(setup_time)}')
    if setup_time < version_time:
        ok = False
    extension_time = check_extensions()
-    log.debug(f'Latest extensions time: {time.ctime(extension_time)}')
+    log.debug(f'Timestamp latest extensions time: {time.ctime(extension_time)}')
    if setup_time < extension_time:
        ok = False
-    log.debug(f'Timestamps: version:{version_time} setup:{setup_time} extension:{extension_time}')
+    log.debug(f'Timestamp: version:{version_time} setup:{setup_time} extension:{extension_time}')
    if args.reinstall:
        ok = False
    return ok
@@ -1226,6 +1291,7 @@ def extensions_preload(parser):
    except Exception:
        log.error('Error running extension preloading')
    if args.profile:
+        pr.disable()
        print_profile(pr, 'Preload')


@@ -31,7 +31,7 @@ const contextMenuInit = () => {
    if ((windowHeight - posy) < menuHeight) contextMenu.style.top = `${windowHeight - menuHeight}px`;
  }

-  function appendContextMenuOption(targetElementSelector, entryName, entryFunction) {
+  function appendContextMenuOption(targetElementSelector, entryName, entryFunction, primary = false) {
    let currentItems = menuSpecs.get(targetElementSelector);
    if (!currentItems) {
      currentItems = [];
@@ -41,7 +41,8 @@ const contextMenuInit = () => {
      id: `${targetElementSelector}_${uid()}`,
      name: entryName,
      func: entryFunction,
-      isNew: true,
+      primary,
+      // isNew: true,
    };
    currentItems.push(newItem);
    return newItem.id;
@@ -64,13 +65,21 @@ const contextMenuInit = () => {
      if (!e.isTrusted) return;
      const oldMenu = gradioApp().querySelector('#context-menu');
      if (oldMenu) oldMenu.remove();
+      menuSpecs.forEach((v, k) => {
+        const items = v.filter((item) => item.primary);
+        if (items.length > 0 && e.composedPath()[0].matches(k)) {
+          showContextMenu(e, e.composedPath()[0], items);
+          e.preventDefault();
+        }
+      });
    });
    gradioApp().addEventListener('contextmenu', (e) => {
      const oldMenu = gradioApp().querySelector('#context-menu');
      if (oldMenu) oldMenu.remove();
      menuSpecs.forEach((v, k) => {
-        if (e.composedPath()[0].matches(k)) {
-          showContextMenu(e, e.composedPath()[0], v);
+        const items = v.filter((item) => !item.primary);
+        if (items.length > 0 && e.composedPath()[0].matches(k)) {
+          showContextMenu(e, e.composedPath()[0], items);
          e.preventDefault();
        }
      });
@@ -80,10 +89,10 @@ const contextMenuInit = () => {
  return [appendContextMenuOption, removeContextMenuOption, addContextMenuEventListener];
 };

-const initResponse = contextMenuInit();
-const appendContextMenuOption = initResponse[0];
-const removeContextMenuOption = initResponse[1];
-const addContextMenuEventListener = initResponse[2];
+const initContextResponse = contextMenuInit();
+const appendContextMenuOption = initContextResponse[0];
+const removeContextMenuOption = initContextResponse[1];
+const addContextMenuEventListener = initContextResponse[2];

 const generateForever = (genbuttonid) => {
  if (window.generateOnRepeatInterval) {
@@ -102,16 +111,25 @@ const generateForever = (genbuttonid) => {
  }
 };

+const reprocessClick = (tabId, state) => {
+  const btn = document.getElementById(`${tabId}_${state}`);
+  window.submit_state = state;
+  if (btn) btn.click();
+};
+
 async function initContextMenu() {
+  let id = '';
  for (const tab of ['txt2img', 'img2img', 'control']) {
-    for (const el of ['generate', 'interrupt', 'skip', 'pause', 'paste', 'clear_prompt', 'extra_networks_btn']) {
-      const id = `#${tab}_${el}`;
-      appendContextMenuOption(id, 'Copy to clipboard', () => navigator.clipboard.writeText(document.querySelector(`#${tab}_prompt > label > textarea`).value));
-      appendContextMenuOption(id, 'Generate forever', () => generateForever(`#${tab}_generate`));
-      appendContextMenuOption(id, 'Apply selected style', quickApplyStyle);
-      appendContextMenuOption(id, 'Quick save style', quickSaveStyle);
-      appendContextMenuOption(id, 'nVidia overlay', initNVML);
-    }
+    id = `#${tab}_generate`;
+    appendContextMenuOption(id, 'Copy to clipboard', () => navigator.clipboard.writeText(document.querySelector(`#${tab}_prompt > label > textarea`).value));
+    appendContextMenuOption(id, 'Generate forever', () => generateForever(`#${tab}_generate`));
+    appendContextMenuOption(id, 'Apply selected style', quickApplyStyle);
+    appendContextMenuOption(id, 'Quick save style', quickSaveStyle);
+    appendContextMenuOption(id, 'nVidia overlay', initNVML);
+    id = `#${tab}_reprocess`;
+    appendContextMenuOption(id, 'Decode full quality', () => reprocessClick(`${tab}`, 'reprocess_decode'), true);
+    appendContextMenuOption(id, 'Refine & HiRes pass', () => reprocessClick(`${tab}`, 'reprocess_refine'), true);
+    appendContextMenuOption(id, 'Detailer pass', () => reprocessClick(`${tab}`, 'reprocess_detail'), true);
  }
  addContextMenuEventListener();
 }
@@ -1,4 +1,6 @@
 function controlInputMode(inputMode, ...args) {
+  const updateEl = gradioApp().getElementById('control_update');
+  if (updateEl) updateEl.click();
  const tab = gradioApp().querySelector('#control-tab-input button.selected');
  if (!tab) return ['Select', ...args];
  inputMode = tab.innerText;
@@ -115,8 +115,6 @@ button.selected {background: var(--button-primary-background-fill);}
 #txt2img_checkboxes, #img2img_checkboxes { background-color: transparent; }
 #txt2img_checkboxes, #img2img_checkboxes { margin-bottom: 0.2em; }
 #txt2img_actions_column, #img2img_actions_column { flex-flow: wrap; justify-content: space-between; }
-#txt2img_enqueue_wrapper, #img2img_enqueue_wrapper { min-width: unset; width: 48%; }
-#txt2img_generate_box, #img2img_generate_box { min-width: unset; width: 48%; }

 #extras_upscale { margin-top: 10px }
 #txt2img_progress_row > div { min-width: var(--left-column); max-width: var(--left-column); }
@@ -328,6 +328,13 @@ function quickSaveStyle() {
  }
 }

+function selectHistory(id) {
+  const headers = new Headers();
+  headers.set('Content-Type', 'application/json');
+  const init = { method: 'POST', body: { name: id }, headers };
+  fetch('/sdapi/v1/history', { method: 'POST', body: JSON.stringify({ name: id }), headers });
+}
+
 let enDirty = false;
 function closeDetailsEN(...args) {
  // log('closeDetailsEN');
@@ -276,33 +276,34 @@ async function gallerySort(btn) {
  const arr = Array.from(el.files.children).filter((node) => node.name); // filter out separators
  const fragment = document.createDocumentFragment();
  el.files.innerHTML = '';
+  log('gallerySort', btn.charCodeAt(0));
  switch (btn.charCodeAt(0)) {
-    case 61789:
+    case 61789: // name asc
      arr
        .sort((a, b) => a.name.localeCompare(b.name))
        .forEach((node) => fragment.appendChild(node));
      break;
-    case 61790:
+    case 61790: // name dsc
      arr
        .sort((b, a) => a.name.localeCompare(b.name))
        .forEach((node) => fragment.appendChild(node));
      break;
-    case 61792:
+    case 61792: // size asc
      arr
        .sort((a, b) => a.size - b.size)
        .forEach((node) => fragment.appendChild(node));
      break;
-    case 61793:
+    case 61793: // size dsc
      arr
        .sort((b, a) => a.size - b.size)
        .forEach((node) => fragment.appendChild(node));
      break;
-    case 61794:
+    case 61794: // resolution asc
      arr
        .sort((a, b) => a.width * a.height - b.width * b.height)
        .forEach((node) => fragment.appendChild(node));
      break;
-    case 61795:
+    case 61795: // resolution dsc
      arr
        .sort((b, a) => a.width * a.height - b.width * b.height)
        .forEach((node) => fragment.appendChild(node));
@@ -1,20 +1,22 @@
 // attaches listeners to the txt2img and img2img galleries to update displayed generation param text when the image changes

-function attachGalleryListeners(tab_name) {
-  const gallery = gradioApp().querySelector(`#${tab_name}_gallery`);
+function attachGalleryListeners(tabName) {
+  const gallery = gradioApp().querySelector(`#${tabName}_gallery`);
  if (!gallery) return null;
-  gallery.addEventListener('click', () => setTimeout(() => {
-    log('galleryItemSelected:', tab_name);
-    gradioApp().getElementById(`${tab_name}_generation_info_button`)?.click();
-  }, 500));
+  gallery.addEventListener('click', () => {
+    // log('galleryItemSelected:', tabName);
+    const btn = gradioApp().getElementById(`${tabName}_generation_info_button`);
+    if (btn) btn.click();
+  });
  gallery?.addEventListener('keydown', (e) => {
-    if (e.keyCode === 37 || e.keyCode === 39) gradioApp().getElementById(`${tab_name}_generation_info_button`).click(); // left or right arrow
+    if (e.keyCode === 37 || e.keyCode === 39) gradioApp().getElementById(`${tabName}_generation_info_button`).click(); // left or right arrow
  });
  return gallery;
 }

 let txt2img_gallery;
 let img2img_gallery;
+let control_gallery;
 let modal;

 async function initiGenerationParams() {
@@ -23,14 +25,17 @@ async function initiGenerationParams() {

  const modalObserver = new MutationObserver((mutations) => {
    mutations.forEach((mutationRecord) => {
-      let selectedTab = gradioApp().querySelector('#tabs div button.selected')?.innerText;
-      if (!selectedTab) selectedTab = gradioApp().querySelector('#tabs div button')?.innerText;
-      if (mutationRecord.target.style.display === 'none' && (selectedTab === 'txt2img' || selectedTab === 'img2img')) { gradioApp().getElementById(`${selectedTab}_generation_info_button`)?.click(); }
+      const tabName = getENActiveTab();
+      if (mutationRecord.target.style.display === 'none') {
+        const btn = gradioApp().getElementById(`${tabName}_generation_info_button`);
+        if (btn) btn.click();
+      }
    });
  });

  if (!txt2img_gallery) txt2img_gallery = attachGalleryListeners('txt2img');
  if (!img2img_gallery) img2img_gallery = attachGalleryListeners('img2img');
+  if (!control_gallery) control_gallery = attachGalleryListeners('control');
  modalObserver.observe(modal, { attributes: true, attributeFilter: ['style'] });
  log('initGenerationParams');
 }
@@ -65,8 +65,8 @@ async function getExif(el) {
  // let html = `<b>Image</b> <a href="${el.src}" target="_blank">${el.src}</a> <b>Size</b> ${el.naturalWidth}x${el.naturalHeight}<br>`;
  let html = '';
  let params;
-  if (exif.paramters) {
-    params = exif.paramters;
+  if (exif.parameters) {
+    params = exif.parameters;
  } else if (exif.userComment) {
    params = Array.from(exif.userComment)
      .map((c) => String.fromCharCode(c))
@@ -159,6 +159,16 @@ function modalResetInstance(event) {
  previewInstance = panzoom(modalImage, { zoomSpeed: 0.05, minZoom: 0.1, maxZoom: 5.0, filterKey: (/* e, dx, dy, dz */) => true });
 }

+function modalToggleParams(event) {
+  const modalExif = gradioApp().getElementById('modalExif');
+  if (modalExif.style.display === 'none' || modalExif.style.display === '') {
+    modalExif.style.display = 'block';
+  } else {
+    modalExif.style.display = 'none';
+  }
+  event.stopPropagation();
+}
+
 function galleryClickEventHandler(event) {
  if (event.button !== 0) return;
  if (event.target.nodeName === 'IMG' && !event.target.parentNode.classList.contains('thumbnail-item')) {
@@ -235,10 +245,17 @@ async function initImageViewer() {
  modalClose.title = 'Close';
  modalClose.addEventListener('click', (evt) => closeModal(evt, true), true);

+  const modalToggleParamsBtn = document.createElement('span');
+  modalToggleParamsBtn.id = 'modal_toggle_params';
+  modalToggleParamsBtn.className = 'cursor';
+  modalToggleParamsBtn.innerHTML = '\uf05a';
+  modalToggleParamsBtn.title = 'Toggle Parameters';
+  modalToggleParamsBtn.addEventListener('click', modalToggleParams, true);
+
  // exif
  const modalExif = document.createElement('div');
  modalExif.id = 'modalExif';
-  modalExif.style = 'position: absolute; bottom: 0px; width: 100%; background-color: rgba(0, 0, 0, 0.5); color: var(--neutral-300); padding: 1em; font-size: small; line-height: 1.2em; z-index: 1';
+  modalExif.style = 'position: absolute; bottom: 0px; width: 100%; background-color: rgba(0, 0, 0, 0.5); color: var(--neutral-300); padding: 1em; font-size: small; line-height: 1.2em; z-index: 1; display: none;';

  // handlers
  modalPreviewZone.addEventListener('mousedown', () => { previewDrag = false; });
@@ -269,13 +286,14 @@ async function initImageViewer() {
  modal.appendChild(modalPreviewZone);
  modal.appendChild(modalNext);
  modal.append(modalControls);
-  modal.append(modalExif);
  modalControls.appendChild(modalZoom);
  modalControls.appendChild(modalReset);
  modalControls.appendChild(modalTile);
  modalControls.appendChild(modalSave);
  modalControls.appendChild(modalDownload);
+  modalControls.appendChild(modalToggleParamsBtn);
  modalControls.appendChild(modalClose);
+  modal.append(modalExif);

  gradioApp().appendChild(modal);
  log('initImageViewer');
@@ -115,8 +115,6 @@ button.selected {background: var(--button-primary-background-fill);}
 #txt2img_checkboxes, #img2img_checkboxes { background-color: transparent; }
 #txt2img_checkboxes, #img2img_checkboxes { margin-bottom: 0.2em; }
 #txt2img_actions_column, #img2img_actions_column { flex-flow: wrap; justify-content: space-between; }
-#txt2img_enqueue_wrapper, #img2img_enqueue_wrapper { min-width: unset; width: 48%; }
-#txt2img_generate_box, #img2img_generate_box { min-width: unset; width: 48%; }

 #extras_upscale { margin-top: 10px }
 #txt2img_progress_row > div { min-width: var(--left-column); max-width: var(--left-column); }
@@ -42,7 +42,8 @@ function checkPaused(state) {
 function setProgress(res) {
  const elements = ['txt2img_generate', 'img2img_generate', 'extras_generate', 'control_generate'];
  const progress = (res?.progress || 0);
-  const job = res?.job || '';
+  let job = res?.job || '';
+  job = job.replace('txt2img', 'Generate').replace('img2img', 'Generate');
  const perc = res && (progress > 0) ? `${Math.round(100.0 * progress)}%` : '';
  let sec = res?.eta || 0;
  let eta = '';
@@ -1,6 +1,6 @@
@font-face { font-family: 'NotoSans'; font-display: swap; font-style: normal; font-weight: 100; src: local('NotoSans'), url('notosans-nerdfont-regular.ttf') }
 :root {
-  --left-column: 520px;
+  --left-column: 530px;
  --color-trace: #666666;
  --color-debug: #7F7F7F;
  --color-info: #D4D4D4;
@@ -17,9 +17,10 @@ textarea { overflow-y: auto !important; }
 span { font-size: var(--text-md) !important; }
 button { font-size: var(--text-lg) !important; }
 input[type='color'] { width: 64px; height: 32px; }
+td > div > span { overflow-y: auto; max-height: 3em; overflow-x: hidden; }

 /* gradio elements */
-.block .padded:not(.gradio-accordion) { padding: 0 !important; margin-right: 0; min-width: 90px !important; }
+.block .padded:not(.gradio-accordion) { padding: 4px 0 0 0 !important; margin-right: 0; min-width: 90px !important; }
 .compact { gap: 1em 0.2em; background: transparent !important; padding: 0 !important; }
 .flex-break { flex-basis: 100% !important; }
 .form { border-width: 0; box-shadow: none; background: transparent; overflow: visible; gap: 0.5em 1em; flex-grow: 1 !important; }
@@ -36,7 +37,7 @@ input[type='color'] { width: 64px; height: 32px; }
 .gradio-button.secondary-down { background: var(--button-secondary-background-fill); color: var(--button-secondary-text-color); }
 .gradio-button.secondary-down, .gradio-button.secondary-down:hover { box-shadow: 1px 1px 1px rgba(0,0,0,0.25) inset, 0px 0px 3px rgba(0,0,0,0.15) inset; }
 .gradio-button.secondary-down:hover { background: var(--button-secondary-background-fill-hover); color: var(--button-secondary-text-color-hover); }
-.gradio-button.tool { max-width: min-content; min-width: min-content !important; align-self: end; font-size: 20px !important; color: var(--body-text-color) !important; align-self: end; }
+.gradio-button.tool { max-width: min-content; min-width: min-content !important; font-size: 20px !important; color: var(--body-text-color) !important; align-self: end; margin-bottom: 4px; }
 .gradio-checkbox { margin: 0.75em 1.5em 0 0; align-self: center; }
 .gradio-column { min-width: min(160px, 100%) !important; }
 .gradio-container { max-width: unset !important; padding: var(--block-label-padding) !important; }
@@ -45,7 +46,7 @@ input[type='color'] { width: 64px; height: 32px; }
 .gradio-dropdown ul.options { z-index: 1000; min-width: fit-content; max-height: 50vh !important; white-space: nowrap; }
 .gradio-dropdown ul.options li.item { padding: var(--spacing-xs); }
 .gradio-dropdown ul.options li.item:not(:has(.hide)) { background-color: var(--primary-500); }
-.gradio-dropdown .token { padding: var(--spacing-xs) !important; }
+.gradio-dropdown .token { padding: var(--spacing-xs) !important; overflow-x: hidden; }
 .gradio-html { color: var(--body-text-color); }
 .gradio-html .min { min-height: 0; }
 .gradio-html div.wrap { height: 100%; }
@@ -54,6 +55,8 @@ input[type='color'] { width: 64px; height: 32px; }
 .gradio-radio { padding: 0 !important; width: max-content !important; }
 .gradio-slider { margin-right: var(--spacing-sm) !important; width: max-content !important }
 .gradio-slider input[type="number"] { width: 5em; font-size: var(--text-xs); height: 16px; text-align: right; padding: 0; }
+.gradio-checkboxgroup { padding: 0 !important; }
+.gradio-checkbox > label { color: var(--block-title-text-color) !important; }

 /* custom gradio elements */
 .accordion-compact { padding: 8px 0px 4px 0px !important; }
@@ -73,7 +76,7 @@ button.custom-button { border-radius: var(--button-large-radius); padding: var(-
 .theme-preview { display: none; position: fixed; border: var(--spacing-sm) solid var(--neutral-600); box-shadow: 2px 2px 2px 2px var(--neutral-700); top: 0; bottom: 0; left: 0; right: 0; margin: auto; max-width: 75vw; z-index: 999; }

 /* txt2img/img2img specific */
-.block.token-counter{ position: absolute; display: inline-block; right: 1em; min-width: 0 !important; width: auto; z-index: 100; top: -0.5em; }
+.block.token-counter{ position: absolute; right: 1em; min-width: 0 !important; width: auto; z-index: 100; top: -0.5em; }
 .block.token-counter span{ background: var(--input-background-fill) !important; box-shadow: 0 0 0.0 0.3em rgba(192,192,192,0.15), inset 0 0 0.6em rgba(192,192,192,0.075); border: 2px solid rgba(192,192,192,0.4) !important; }
 .block.token-counter.error span{ box-shadow: 0 0 0.0 0.3em rgba(255,0,0,0.15), inset 0 0 0.6em rgba(255,0,0,0.075); border: 2px solid rgba(255,0,0,0.4) !important; }
 .block.token-counter div{ display: inline; }
@@ -83,6 +86,7 @@ button.custom-button { border-radius: var(--button-large-radius); padding: var(-
 .performance .time { margin-right: 0; }
 .thumbnails { background: var(--body-background-fill); }
 .control-image { height: calc(100vw/3) !important; }
+.prompt textarea { resize: vertical; }
 #control_results { margin: 0; padding: 0; }
 #control_gallery { height: calc(100vw/3 + 60px); }
 #txt2img_gallery, #img2img_gallery { height: 50vh; }
@@ -90,17 +94,17 @@ button.custom-button { border-radius: var(--button-large-radius); padding: var(-
 #control-inputs { margin-top: 1em; }
 #txt2img_prompt_container, #img2img_prompt_container, #control_prompt_container { margin-right: var(--layout-gap) }
 #txt2img_footer, #img2img_footer, #control_footer { height: fit-content; display: none; }
-#txt2img_generate_box, #img2img_generate_box, #control_general_box { gap: 0.5em; flex-wrap: wrap-reverse; height: fit-content; }
+#txt2img_generate_box, #img2img_generate_box, #control_generate_box { gap: 0.5em; flex-wrap: unset; min-width: unset; width: 66.6%; }
 #txt2img_actions_column, #img2img_actions_column, #control_actions_column { gap: 0.3em; height: fit-content; }
-#txt2img_generate_box>button, #img2img_generate_box>button, #control_generate_box>button, #txt2img_enqueue, #img2img_enqueue { min-height: 44px !important; max-height: 44px !important; line-height: 1em; }
+#txt2img_generate_box>button, #img2img_generate_box>button, #control_generate_box>button, #txt2img_enqueue, #img2img_enqueue, #txt2img_enqueue>button, #img2img_enqueue>button { min-height: 44px !important; max-height: 44px !important; line-height: 1em; white-space: break-spaces; min-width: unset; }
+#txt2img_enqueue_wrapper, #img2img_enqueue_wrapper, #control_enqueue_wrapper { min-width: unset !important; width: 31%; }
 #txt2img_generate_line2, #img2img_generate_line2, #txt2img_tools, #img2img_tools, #control_generate_line2, #control_tools { display: flex; }
 #txt2img_generate_line2>button, #img2img_generate_line2>button, #extras_generate_box>button, #control_generate_line2>button, #txt2img_tools>button, #img2img_tools>button, #control_tools>button { height: 2em; line-height: 0; font-size: var(--text-md);
  min-width: unset; display: block !important; }
 #txt2img_prompt, #txt2img_neg_prompt, #img2img_prompt, #img2img_neg_prompt, #control_prompt, #control_neg_prompt { display: contents; }
-#txt2img_generate_box, #img2img_generate_box { min-width: unset; width: 66%; }
-#control_generate_box { min-width: unset; width: 100%; }
 #txt2img_actions_column, #img2img_actions_column, #control_actions { flex-flow: wrap; justify-content: space-between; }
-#txt2img_enqueue_wrapper, #img2img_enqueue_wrapper, #control_enqueue_wrapper { min-width: unset !important; width: 32%; }
+
+
 .interrogate-clip { position: absolute; right: 6em; top: 8px; max-width: fit-content; background: none !important; z-index: 50; }
 .interrogate-blip { position: absolute; right: 4em; top: 8px; max-width: fit-content; background: none !important; z-index: 50; }
 .interrogate-col { min-width: 0 !important; max-width: fit-content; margin-right: var(--spacing-xxl); }
@@ -224,6 +228,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
 .extra-network-cards .card .overlay .tag { padding: 2px; margin: 2px; background: rgba(70, 70, 70, 0.60); font-size: var(--text-md); cursor: pointer; display: inline-block; }
 .extra-network-cards .card .actions>span { padding: 4px; font-size: 34px !important; }
 .extra-network-cards .card .actions>span:hover { color: var(--highlight-color); }
+.extra-network-cards .card .version { position: absolute; top: 0; left: 0; padding: 2px; font-weight: bolder; text-shadow: 1px 1px black; text-transform: uppercase; background: gray; opacity: 75%; margin: 4px; line-height: 0.9rem; }
 .extra-network-cards .card:hover .actions { display: block; }
 .extra-network-cards .card:hover .overlay .tags { display: block; }
 .extra-network-cards .card:has(>img[src*="card-no-preview.png"])::before { content: ''; position: absolute; width: 100%; height: 100%; mix-blend-mode: multiply; background-color: var(--data-color); }
@@ -7,7 +7,6 @@ async function initStartup() {
  // all items here are non-blocking async calls
  initModels();
  getUIDefaults();
-  initiGenerationParams();
  initPromptChecker();
  initLogMonitor();
  initContextMenu();
@@ -16,6 +15,7 @@ async function initStartup() {
  initSettings();
  initImageViewer();
  initGallery();
+  initiGenerationParams();
  setupControlUI();

  // reconnect server session
@@ -115,8 +115,6 @@ button.selected {background: var(--button-primary-background-fill);}
 #txt2img_checkboxes, #img2img_checkboxes { background-color: transparent; }
 #txt2img_checkboxes, #img2img_checkboxes { margin-bottom: 0.2em; }
 #txt2img_actions_column, #img2img_actions_column { flex-flow: wrap; justify-content: space-between; }
-#txt2img_enqueue_wrapper, #img2img_enqueue_wrapper { min-width: unset; width: 48%; }
-#txt2img_generate_box, #img2img_generate_box { min-width: unset; width: 48%; }

 #extras_upscale { margin-top: 10px }
 #txt2img_progress_row > div { min-width: var(--left-column); max-width: var(--left-column); }
@@ -204,6 +204,8 @@ function submit_txt2img(...args) {
  requestProgress(id, null, gradioApp().getElementById('txt2img_gallery'));
  const res = create_submit_args(args);
  res[0] = id;
+  res[1] = window.submit_state;
+  window.submit_state = '';
  return res;
 }

@@ -214,7 +216,9 @@ function submit_img2img(...args) {
  requestProgress(id, null, gradioApp().getElementById('img2img_gallery'));
  const res = create_submit_args(args);
  res[0] = id;
-  res[1] = get_tab_index('mode_img2img');
+  res[1] = window.submit_state;
+  res[2] = get_tab_index('mode_img2img');
+  window.submit_state = '';
  return res;
 }

@@ -225,7 +229,9 @@ function submit_control(...args) {
  requestProgress(id, null, gradioApp().getElementById('control_gallery'));
  const res = create_submit_args(args);
  res[0] = id;
-  res[1] = gradioApp().querySelector('#control-tabs > .tab-nav > .selected')?.innerText.toLowerCase() || ''; // selected tab name
+  res[1] = window.submit_state;
+  res[2] = gradioApp().querySelector('#control-tabs > .tab-nav > .selected')?.innerText.toLowerCase() || ''; // selected tab name
+  window.submit_state = '';
  return res;
 }

@@ -236,6 +242,7 @@ function submit_postprocessing(...args) {
 }

 window.submit = submit_txt2img;
+window.submit_state = '';

 function modelmerger(...args) {
  const id = randomId();
@@ -169,6 +169,12 @@ def start_server(immediate=True, server=None):
    uvicorn = None
    if args.test:
        installer.log.info("Test only")
+        installer.log.critical('Logging: level=critical')
+        installer.log.error('Logging: level=error')
+        installer.log.warning('Logging: level=warning')
+        installer.log.info('Logging: level=info')
+        installer.log.debug('Logging: level=debug')
+        installer.log.trace('Logging: level=trace')
        server.wants_restart = False
    else:
        if args.api_only:
@@ -176,6 +182,7 @@ def start_server(immediate=True, server=None):
        else:
            uvicorn = server.webui(restart=not immediate)
    if args.profile:
+        pr.disable()
        installer.print_profile(pr, 'WebUI')
    return uvicorn, server

@@ -208,6 +215,7 @@ def main():
        installer.install("uv", "uv")
    installer.check_torch()
    installer.check_onnx()
+    installer.check_torchao()
    installer.check_diffusers()
    installer.check_modified_files()
    if args.reinstall:
@@ -0,0 +1,58 @@
+# copied from paper: <https://arxiv.org/pdf/2410.02416>
+
+import torch
+import diffusers
+from .pipeline_stable_diffision_xl_apg import StableDiffusionXLPipelineAPG
+from .pipeline_stable_cascade_prior_apg import StableCascadePriorPipelineAPG
+from .pipeline_stable_diffusion_apg import StableDiffusionPipelineAPG
+
+
+class MomentumBuffer:
+    def __init__(self, momentum_val: float):
+        self.momentum = momentum_val
+        self.running_average = 0
+    def update(self, update_value: torch.Tensor):
+        new_average = self.momentum * self.running_average
+        self.running_average = update_value + new_average
+
+
+eta = 0
+momentum = 0
+threshold = 0
+buffer: MomentumBuffer = None
+orig_pipe: diffusers.DiffusionPipeline = None
+
+
+def project(
+    v0: torch.Tensor, # [B, C, H, W]
+    v1: torch.Tensor, # [B, C, H, W]
+    ):
+    device = v0.device
+    dtype = v0.dtype
+    if device.type == "xpu":
+        v0, v1 = v0.to("cpu"), v1.to("cpu")
+    v0, v1 = v0.double(), v1.double()
+    v1 = torch.nn.functional.normalize(v1, dim=[-1, -2, -3])
+    v0_parallel = (v0 * v1).sum(dim=[-1, -2, -3], keepdim=True) * v1
+    v0_orthogonal = v0 - v0_parallel
+    return v0_parallel.to(device, dtype=dtype), v0_orthogonal.to(device, dtype=dtype)
+
+
+def normalized_guidance(
+    pred_cond: torch.Tensor, # [B, C, H, W]
+    pred_uncond: torch.Tensor, # [B, C, H, W]
+    guidance_scale: float,
+    ):
+    diff = pred_cond - pred_uncond
+    if buffer is not None:
+        buffer.update(diff)
+        diff = buffer.running_average
+    if threshold > 0:
+        ones = torch.ones_like(diff)
+        diff_norm = diff.norm(p=2, dim=[-1, -2, -3], keepdim=True)
+        scale_factor = torch.minimum(ones, threshold / diff_norm)
+        diff = diff * scale_factor
+    diff_parallel, diff_orthogonal = project(diff, pred_cond)
+    normalized_update = diff_orthogonal + eta * diff_parallel
+    pred_guided = pred_cond + (guidance_scale - 1) * normalized_update
+    return pred_guided
@@ -0,0 +1,638 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass
+from math import ceil
+from typing import Callable, Dict, List, Optional, Union
+
+import numpy as np
+import PIL
+import torch
+from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection
+
+from diffusers.models import StableCascadeUNet
+from diffusers.schedulers import DDPMWuerstchenScheduler
+from diffusers.utils import BaseOutput, logging, replace_example_docstring
+from diffusers.utils.torch_utils import randn_tensor
+from diffusers.pipelines.pipeline_utils import DiffusionPipeline
+from modules import apg
+
+
+logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+
+DEFAULT_STAGE_C_TIMESTEPS = list(np.linspace(1.0, 2 / 3, 20)) + list(np.linspace(2 / 3, 0.0, 11))[1:]
+
+EXAMPLE_DOC_STRING = """
+    Examples:
+        ```py
+        >>> import torch
+        >>> from diffusers import StableCascadePriorPipeline
+
+        >>> prior_pipe = StableCascadePriorPipeline.from_pretrained(
+        ...     "stabilityai/stable-cascade-prior", torch_dtype=torch.bfloat16
+        ... ).to("cuda")
+
+        >>> prompt = "an image of a shiba inu, donning a spacesuit and helmet"
+        >>> prior_output = pipe(prompt)
+        ```
+"""
+
+
+@dataclass
+class StableCascadePriorPipelineOutput(BaseOutput):
+    """
+    Output class for WuerstchenPriorPipeline.
+
+    Args:
+        image_embeddings (`torch.Tensor` or `np.ndarray`)
+            Prior image embeddings for text prompt
+        prompt_embeds (`torch.Tensor`):
+            Text embeddings for the prompt.
+        negative_prompt_embeds (`torch.Tensor`):
+            Text embeddings for the negative prompt.
+    """
+
+    image_embeddings: Union[torch.Tensor, np.ndarray]
+    prompt_embeds: Union[torch.Tensor, np.ndarray]
+    prompt_embeds_pooled: Union[torch.Tensor, np.ndarray]
+    negative_prompt_embeds: Union[torch.Tensor, np.ndarray]
+    negative_prompt_embeds_pooled: Union[torch.Tensor, np.ndarray]
+
+
+class StableCascadePriorPipelineAPG(DiffusionPipeline):
+    """
+    Pipeline for generating image prior for Stable Cascade.
+
+    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
+    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
+
+    Args:
+        prior ([`StableCascadeUNet`]):
+            The Stable Cascade prior to approximate the image embedding from the text and/or image embedding.
+        text_encoder ([`CLIPTextModelWithProjection`]):
+            Frozen text-encoder
+            ([laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)).
+        feature_extractor ([`~transformers.CLIPImageProcessor`]):
+            Model that extracts features from generated images to be used as inputs for the `image_encoder`.
+        image_encoder ([`CLIPVisionModelWithProjection`]):
+            Frozen CLIP image-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).
+        tokenizer (`CLIPTokenizer`):
+            Tokenizer of class
+            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
+        scheduler ([`DDPMWuerstchenScheduler`]):
+            A scheduler to be used in combination with `prior` to generate image embedding.
+        resolution_multiple ('float', *optional*, defaults to 42.67):
+            Default resolution for multiple images generated.
+    """
+
+    unet_name = "prior"
+    text_encoder_name = "text_encoder"
+    model_cpu_offload_seq = "image_encoder->text_encoder->prior"
+    _optional_components = ["image_encoder", "feature_extractor"]
+    _callback_tensor_inputs = ["latents", "text_encoder_hidden_states", "negative_prompt_embeds"]
+
+    def __init__(
+        self,
+        tokenizer: CLIPTokenizer,
+        text_encoder: CLIPTextModelWithProjection,
+        prior: StableCascadeUNet,
+        scheduler: DDPMWuerstchenScheduler,
+        resolution_multiple: float = 42.67,
+        feature_extractor: Optional[CLIPImageProcessor] = None,
+        image_encoder: Optional[CLIPVisionModelWithProjection] = None,
+    ) -> None:
+        super().__init__()
+        self.register_modules(
+            tokenizer=tokenizer,
+            text_encoder=text_encoder,
+            image_encoder=image_encoder,
+            feature_extractor=feature_extractor,
+            prior=prior,
+            scheduler=scheduler,
+        )
+        self.register_to_config(resolution_multiple=resolution_multiple)
+
+    def prepare_latents(
+        self, batch_size, height, width, num_images_per_prompt, dtype, device, generator, latents, scheduler
+    ):
+        latent_shape = (
+            num_images_per_prompt * batch_size,
+            self.prior.config.in_channels,
+            ceil(height / self.config.resolution_multiple),
+            ceil(width / self.config.resolution_multiple),
+        )
+
+        if latents is None:
+            latents = randn_tensor(latent_shape, generator=generator, device=device, dtype=dtype)
+        else:
+            if latents.shape != latent_shape:
+                raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latent_shape}")
+            latents = latents.to(device)
+
+        latents = latents * scheduler.init_noise_sigma
+        return latents
+
+    def encode_prompt(
+        self,
+        device,
+        batch_size,
+        num_images_per_prompt,
+        do_classifier_free_guidance,
+        prompt=None,
+        negative_prompt=None,
+        prompt_embeds: Optional[torch.Tensor] = None,
+        prompt_embeds_pooled: Optional[torch.Tensor] = None,
+        negative_prompt_embeds: Optional[torch.Tensor] = None,
+        negative_prompt_embeds_pooled: Optional[torch.Tensor] = None,
+    ):
+        if prompt_embeds is None:
+            # get prompt text embeddings
+            text_inputs = self.tokenizer(
+                prompt,
+                padding="max_length",
+                max_length=self.tokenizer.model_max_length,
+                truncation=True,
+                return_tensors="pt",
+            )
+            text_input_ids = text_inputs.input_ids
+            attention_mask = text_inputs.attention_mask
+
+            untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
+
+            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
+                text_input_ids, untruncated_ids
+            ):
+                removed_text = self.tokenizer.batch_decode(
+                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]
+                )
+                logger.warning(
+                    "The following part of your input was truncated because CLIP can only handle sequences up to"
+                    f" {self.tokenizer.model_max_length} tokens: {removed_text}"
+                )
+                text_input_ids = text_input_ids[:, : self.tokenizer.model_max_length]
+                attention_mask = attention_mask[:, : self.tokenizer.model_max_length]
+
+            text_encoder_output = self.text_encoder(
+                text_input_ids.to(device), attention_mask=attention_mask.to(device), output_hidden_states=True
+            )
+            prompt_embeds = text_encoder_output.hidden_states[-1]
+            if prompt_embeds_pooled is None:
+                prompt_embeds_pooled = text_encoder_output.text_embeds.unsqueeze(1)
+
+        prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
+        prompt_embeds_pooled = prompt_embeds_pooled.to(dtype=self.text_encoder.dtype, device=device)
+        prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0)
+        prompt_embeds_pooled = prompt_embeds_pooled.repeat_interleave(num_images_per_prompt, dim=0)
+
+        if negative_prompt_embeds is None and do_classifier_free_guidance:
+            uncond_tokens: List[str]
+            if negative_prompt is None:
+                uncond_tokens = [""] * batch_size
+            elif type(prompt) is not type(negative_prompt):
+                raise TypeError(
+                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
+                    f" {type(prompt)}."
+                )
+            elif isinstance(negative_prompt, str):
+                uncond_tokens = [negative_prompt]
+            elif batch_size != len(negative_prompt):
+                raise ValueError(
+                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
+                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
+                    " the batch size of `prompt`."
+                )
+            else:
+                uncond_tokens = negative_prompt
+
+            uncond_input = self.tokenizer(
+                uncond_tokens,
+                padding="max_length",
+                max_length=self.tokenizer.model_max_length,
+                truncation=True,
+                return_tensors="pt",
+            )
+            negative_prompt_embeds_text_encoder_output = self.text_encoder(
+                uncond_input.input_ids.to(device),
+                attention_mask=uncond_input.attention_mask.to(device),
+                output_hidden_states=True,
+            )
+
+            negative_prompt_embeds = negative_prompt_embeds_text_encoder_output.hidden_states[-1]
+            negative_prompt_embeds_pooled = negative_prompt_embeds_text_encoder_output.text_embeds.unsqueeze(1)
+
+        if do_classifier_free_guidance:
+            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
+            seq_len = negative_prompt_embeds.shape[1]
+            negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
+            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
+            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
+
+            seq_len = negative_prompt_embeds_pooled.shape[1]
+            negative_prompt_embeds_pooled = negative_prompt_embeds_pooled.to(
+                dtype=self.text_encoder.dtype, device=device
+            )
+            negative_prompt_embeds_pooled = negative_prompt_embeds_pooled.repeat(1, num_images_per_prompt, 1)
+            negative_prompt_embeds_pooled = negative_prompt_embeds_pooled.view(
+                batch_size * num_images_per_prompt, seq_len, -1
+            )
+            # done duplicates
+
+        return prompt_embeds, prompt_embeds_pooled, negative_prompt_embeds, negative_prompt_embeds_pooled
+
+    def encode_image(self, images, device, dtype, batch_size, num_images_per_prompt):
+        image_embeds = []
+        for image in images:
+            image = self.feature_extractor(image, return_tensors="pt").pixel_values
+            image = image.to(device=device, dtype=dtype)
+            image_embed = self.image_encoder(image).image_embeds.unsqueeze(1)
+            image_embeds.append(image_embed)
+        image_embeds = torch.cat(image_embeds, dim=1)
+
+        image_embeds = image_embeds.repeat(batch_size * num_images_per_prompt, 1, 1)
+        negative_image_embeds = torch.zeros_like(image_embeds)
+
+        return image_embeds, negative_image_embeds
+
+    def check_inputs(
+        self,
+        prompt,
+        images=None,
+        image_embeds=None,
+        negative_prompt=None,
+        prompt_embeds=None,
+        prompt_embeds_pooled=None,
+        negative_prompt_embeds=None,
+        negative_prompt_embeds_pooled=None,
+        callback_on_step_end_tensor_inputs=None,
+    ):
+        if callback_on_step_end_tensor_inputs is not None and not all(
+            k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
+        ):
+            raise ValueError(
+                f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
+            )
+
+        if prompt is not None and prompt_embeds is not None:
+            raise ValueError(
+                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
+                " only forward one of the two."
+            )
+        elif prompt is None and prompt_embeds is None:
+            raise ValueError(
+                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
+            )
+        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
+            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
+
+        if negative_prompt is not None and negative_prompt_embeds is not None:
+            raise ValueError(
+                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
+                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
+            )
+
+        if prompt_embeds is not None and negative_prompt_embeds is not None:
+            if prompt_embeds.shape != negative_prompt_embeds.shape:
+                raise ValueError(
+                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
+                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
+                    f" {negative_prompt_embeds.shape}."
+                )
+
+        if prompt_embeds is not None and prompt_embeds_pooled is None:
+            raise ValueError(
+                "If `prompt_embeds` are provided, `prompt_embeds_pooled` must also be provided. Make sure to generate `prompt_embeds_pooled` from the same text encoder that was used to generate `prompt_embeds`"
+            )
+
+        if negative_prompt_embeds is not None and negative_prompt_embeds_pooled is None:
+            raise ValueError(
+                "If `negative_prompt_embeds` are provided, `negative_prompt_embeds_pooled` must also be provided. Make sure to generate `prompt_embeds_pooled` from the same text encoder that was used to generate `prompt_embeds`"
+            )
+
+        if prompt_embeds_pooled is not None and negative_prompt_embeds_pooled is not None:
+            if prompt_embeds_pooled.shape != negative_prompt_embeds_pooled.shape:
+                raise ValueError(
+                    "`prompt_embeds_pooled` and `negative_prompt_embeds_pooled` must have the same shape when passed"
+                    f"directly, but got: `prompt_embeds_pooled` {prompt_embeds_pooled.shape} !="
+                    f"`negative_prompt_embeds_pooled` {negative_prompt_embeds_pooled.shape}."
+                )
+
+        if image_embeds is not None and images is not None:
+            raise ValueError(
+                f"Cannot forward both `images`: {images} and `image_embeds`: {image_embeds}. Please make sure to"
+                " only forward one of the two."
+            )
+
+        if images:
+            for i, image in enumerate(images):
+                if not isinstance(image, torch.Tensor) and not isinstance(image, PIL.Image.Image):
+                    raise TypeError(
+                        f"'images' must contain images of type 'torch.Tensor' or 'PIL.Image.Image, but got"
+                        f"{type(image)} for image number {i}."
+                    )
+
+    @property
+    def guidance_scale(self):
+        return self._guidance_scale
+
+    @property
+    def do_classifier_free_guidance(self):
+        return self._guidance_scale > 1
+
+    @property
+    def num_timesteps(self):
+        return self._num_timesteps
+
+    def get_timestep_ratio_conditioning(self, t, alphas_cumprod):
+        s = torch.tensor([0.008])
+        clamp_range = [0, 1]
+        min_var = torch.cos(s / (1 + s) * torch.pi * 0.5) ** 2
+        var = alphas_cumprod[t]
+        var = var.clamp(*clamp_range)
+        s, min_var = s.to(var.device), min_var.to(var.device)
+        ratio = (((var * min_var) ** 0.5).acos() / (torch.pi * 0.5)) * (1 + s) - s
+        return ratio
+
+    @torch.no_grad()
+    @replace_example_docstring(EXAMPLE_DOC_STRING)
+    def __call__(
+        self,
+        prompt: Optional[Union[str, List[str]]] = None,
+        images: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]] = None,
+        height: int = 1024,
+        width: int = 1024,
+        num_inference_steps: int = 20,
+        timesteps: List[float] = None,
+        guidance_scale: float = 4.0,
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        prompt_embeds: Optional[torch.Tensor] = None,
+        prompt_embeds_pooled: Optional[torch.Tensor] = None,
+        negative_prompt_embeds: Optional[torch.Tensor] = None,
+        negative_prompt_embeds_pooled: Optional[torch.Tensor] = None,
+        image_embeds: Optional[torch.Tensor] = None,
+        num_images_per_prompt: Optional[int] = 1,
+        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+        latents: Optional[torch.Tensor] = None,
+        output_type: Optional[str] = "pt",
+        return_dict: bool = True,
+        callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
+        callback_on_step_end_tensor_inputs: List[str] = ["latents"],
+    ):
+        """
+        Function invoked when calling the pipeline for generation.
+
+        Args:
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            height (`int`, *optional*, defaults to 1024):
+                The height in pixels of the generated image.
+            width (`int`, *optional*, defaults to 1024):
+                The width in pixels of the generated image.
+            num_inference_steps (`int`, *optional*, defaults to 60):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            guidance_scale (`float`, *optional*, defaults to 8.0):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `decoder_guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting
+                `decoder_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely
+                linked to the text `prompt`, usually at the expense of lower image quality.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `decoder_guidance_scale` is less than `1`).
+            prompt_embeds (`torch.Tensor`, *optional*):
+                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
+                provided, text embeddings will be generated from `prompt` input argument.
+            prompt_embeds_pooled (`torch.Tensor`, *optional*):
+                Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
+                If not provided, pooled text embeddings will be generated from `prompt` input argument.
+            negative_prompt_embeds (`torch.Tensor`, *optional*):
+                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
+                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
+                argument.
+            negative_prompt_embeds_pooled (`torch.Tensor`, *optional*):
+                Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
+                weighting. If not provided, negative_prompt_embeds_pooled will be generated from `negative_prompt`
+                input argument.
+            image_embeds (`torch.Tensor`, *optional*):
+                Pre-generated image embeddings. Can be used to easily tweak image inputs, *e.g.* prompt weighting. If
+                not provided, image embeddings will be generated from `image` input argument if existing.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
+                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
+                to make generation deterministic.
+            latents (`torch.Tensor`, *optional*):
+                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
+                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
+                tensor will ge generated by sampling using the supplied random `generator`.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"`
+                (`np.array`) or `"pt"` (`torch.Tensor`).
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple.
+            callback_on_step_end (`Callable`, *optional*):
+                A function that calls at the end of each denoising steps during the inference. The function is called
+                with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
+                callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
+                `callback_on_step_end_tensor_inputs`.
+            callback_on_step_end_tensor_inputs (`List`, *optional*):
+                The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
+                will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
+                `._callback_tensor_inputs` attribute of your pipeline class.
+
+        Examples:
+
+        Returns:
+            [`StableCascadePriorPipelineOutput`] or `tuple` [`StableCascadePriorPipelineOutput`] if `return_dict` is
+            True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated image
+            embeddings.
+        """
+
+        # 0. Define commonly used variables
+        device = self._execution_device
+        dtype = next(self.prior.parameters()).dtype
+        self._guidance_scale = guidance_scale
+        if prompt is not None and isinstance(prompt, str):
+            batch_size = 1
+        elif prompt is not None and isinstance(prompt, list):
+            batch_size = len(prompt)
+        else:
+            batch_size = prompt_embeds.shape[0]
+
+        # 1. Check inputs. Raise error if not correct
+        self.check_inputs(
+            prompt,
+            images=images,
+            image_embeds=image_embeds,
+            negative_prompt=negative_prompt,
+            prompt_embeds=prompt_embeds,
+            prompt_embeds_pooled=prompt_embeds_pooled,
+            negative_prompt_embeds=negative_prompt_embeds,
+            negative_prompt_embeds_pooled=negative_prompt_embeds_pooled,
+            callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
+        )
+
+        # 2. Encode caption + images
+        (
+            prompt_embeds,
+            prompt_embeds_pooled,
+            negative_prompt_embeds,
+            negative_prompt_embeds_pooled,
+        ) = self.encode_prompt(
+            prompt=prompt,
+            device=device,
+            batch_size=batch_size,
+            num_images_per_prompt=num_images_per_prompt,
+            do_classifier_free_guidance=self.do_classifier_free_guidance,
+            negative_prompt=negative_prompt,
+            prompt_embeds=prompt_embeds,
+            prompt_embeds_pooled=prompt_embeds_pooled,
+            negative_prompt_embeds=negative_prompt_embeds,
+            negative_prompt_embeds_pooled=negative_prompt_embeds_pooled,
+        )
+
+        if images is not None:
+            image_embeds_pooled, uncond_image_embeds_pooled = self.encode_image(
+                images=images,
+                device=device,
+                dtype=dtype,
+                batch_size=batch_size,
+                num_images_per_prompt=num_images_per_prompt,
+            )
+        elif image_embeds is not None:
+            image_embeds_pooled = image_embeds.repeat(batch_size * num_images_per_prompt, 1, 1)
+            uncond_image_embeds_pooled = torch.zeros_like(image_embeds_pooled)
+        else:
+            image_embeds_pooled = torch.zeros(
+                batch_size * num_images_per_prompt,
+                1,
+                self.prior.config.clip_image_in_channels,
+                device=device,
+                dtype=dtype,
+            )
+            uncond_image_embeds_pooled = torch.zeros(
+                batch_size * num_images_per_prompt,
+                1,
+                self.prior.config.clip_image_in_channels,
+                device=device,
+                dtype=dtype,
+            )
+
+        if self.do_classifier_free_guidance:
+            image_embeds = torch.cat([image_embeds_pooled, uncond_image_embeds_pooled], dim=0)
+        else:
+            image_embeds = image_embeds_pooled
+
+        # For classifier free guidance, we need to do two forward passes.
+        # Here we concatenate the unconditional and text embeddings into a single batch
+        # to avoid doing two forward passes
+        text_encoder_hidden_states = (
+            torch.cat([prompt_embeds, negative_prompt_embeds]) if negative_prompt_embeds is not None else prompt_embeds
+        )
+        text_encoder_pooled = (
+            torch.cat([prompt_embeds_pooled, negative_prompt_embeds_pooled])
+            if negative_prompt_embeds is not None
+            else prompt_embeds_pooled
+        )
+
+        # 4. Prepare and set timesteps
+        self.scheduler.set_timesteps(num_inference_steps, device=device)
+        timesteps = self.scheduler.timesteps
+
+        # 5. Prepare latents
+        latents = self.prepare_latents(
+            batch_size, height, width, num_images_per_prompt, dtype, device, generator, latents, self.scheduler
+        )
+
+        if isinstance(self.scheduler, DDPMWuerstchenScheduler):
+            timesteps = timesteps[:-1]
+        else:
+            if hasattr(self.scheduler.config, "clip_sample") and self.scheduler.config.clip_sample:
+                self.scheduler.config.clip_sample = False  # disample sample clipping
+                logger.warning(" set `clip_sample` to be False")
+        # 6. Run denoising loop
+        if hasattr(self.scheduler, "betas"):
+            alphas = 1.0 - self.scheduler.betas
+            alphas_cumprod = torch.cumprod(alphas, dim=0)
+        else:
+            alphas_cumprod = []
+
+        self._num_timesteps = len(timesteps)
+        for i, t in enumerate(self.progress_bar(timesteps)):
+            if not isinstance(self.scheduler, DDPMWuerstchenScheduler):
+                if len(alphas_cumprod) > 0:
+                    timestep_ratio = self.get_timestep_ratio_conditioning(t.long().cpu(), alphas_cumprod)
+                    timestep_ratio = timestep_ratio.expand(latents.size(0)).to(dtype).to(device)
+                else:
+                    timestep_ratio = t.float().div(self.scheduler.timesteps[-1]).expand(latents.size(0)).to(dtype)
+            else:
+                timestep_ratio = t.expand(latents.size(0)).to(dtype)
+            # 7. Denoise image embeddings
+            predicted_image_embedding = self.prior(
+                sample=torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents,
+                timestep_ratio=torch.cat([timestep_ratio] * 2) if self.do_classifier_free_guidance else timestep_ratio,
+                clip_text_pooled=text_encoder_pooled,
+                clip_text=text_encoder_hidden_states,
+                clip_img=image_embeds,
+                return_dict=False,
+            )[0]
+
+            # 8. Check for classifier free guidance and apply it
+            if self.do_classifier_free_guidance:
+                predicted_image_embedding_text, predicted_image_embedding_uncond = predicted_image_embedding.chunk(2)
+                predicted_image_embedding = apg.normalized_guidance(predicted_image_embedding_text, predicted_image_embedding_uncond, self.guidance_scale)
+
+            # 9. Renoise latents to next timestep
+            if not isinstance(self.scheduler, DDPMWuerstchenScheduler):
+                timestep_ratio = t
+            latents = self.scheduler.step(
+                model_output=predicted_image_embedding, timestep=timestep_ratio, sample=latents, generator=generator
+            ).prev_sample
+
+            if callback_on_step_end is not None:
+                callback_kwargs = {}
+                for k in callback_on_step_end_tensor_inputs:
+                    callback_kwargs[k] = locals()[k]
+                callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
+
+                latents = callback_outputs.pop("latents", latents)
+                prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
+                negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
+
+        # Offload all models
+        self.maybe_free_model_hooks()
+
+        if output_type == "np":
+            latents = latents.cpu().float().numpy()  # float() as bfloat16-> numpy doesnt work
+            prompt_embeds = prompt_embeds.cpu().float().numpy()  # float() as bfloat16-> numpy doesnt work
+            negative_prompt_embeds = (
+                negative_prompt_embeds.cpu().float().numpy() if negative_prompt_embeds is not None else None
+            )  # float() as bfloat16-> numpy doesnt work
+
+        if not return_dict:
+            return (
+                latents,
+                prompt_embeds,
+                prompt_embeds_pooled,
+                negative_prompt_embeds,
+                negative_prompt_embeds_pooled,
+            )
+
+        return StableCascadePriorPipelineOutput(
+            image_embeddings=latents,
+            prompt_embeds=prompt_embeds,
+            prompt_embeds_pooled=prompt_embeds_pooled,
+            negative_prompt_embeds=negative_prompt_embeds,
+            negative_prompt_embeds_pooled=negative_prompt_embeds_pooled,
+        )
@@ -69,7 +69,7 @@ class Api:
        self.add_api_route("/sdapi/v1/upscalers", endpoints.get_upscalers, methods=["GET"], response_model=List[models.ItemUpscaler])
        self.add_api_route("/sdapi/v1/sd-models", endpoints.get_sd_models, methods=["GET"], response_model=List[models.ItemModel])
        self.add_api_route("/sdapi/v1/hypernetworks", endpoints.get_hypernetworks, methods=["GET"], response_model=List[models.ItemHypernetwork])
-        self.add_api_route("/sdapi/v1/face-restorers", endpoints.get_face_restorers, methods=["GET"], response_model=List[models.ItemFaceRestorer])
+        self.add_api_route("/sdapi/v1/face-restorers", endpoints.get_detailers, methods=["GET"], response_model=List[models.ItemDetailer])
        self.add_api_route("/sdapi/v1/prompt-styles", endpoints.get_prompt_styles, methods=["GET"], response_model=List[models.ItemStyle])
        self.add_api_route("/sdapi/v1/embeddings", endpoints.get_embeddings, methods=["GET"], response_model=models.ResEmbeddings)
        self.add_api_route("/sdapi/v1/sd-vae", endpoints.get_sd_vaes, methods=["GET"], response_model=List[models.ItemVae])
@@ -84,6 +84,8 @@ class Api:
        self.add_api_route("/sdapi/v1/unload-checkpoint", endpoints.post_unload_checkpoint, methods=["POST"])
        self.add_api_route("/sdapi/v1/reload-checkpoint", endpoints.post_reload_checkpoint, methods=["POST"])
        self.add_api_route("/sdapi/v1/refresh-vae", endpoints.post_refresh_vae, methods=["POST"])
+        self.add_api_route("/sdapi/v1/history", endpoints.get_history, methods=["GET"], response_model=List[str])
+        self.add_api_route("/sdapi/v1/history", endpoints.post_history, methods=["POST"], response_model=int)

        # gallery api
        gallery.register_api(app)
@@ -23,8 +23,8 @@ def get_sd_models():
 def get_hypernetworks():
    return [{"name": name, "path": shared.hypernetworks[name]} for name in shared.hypernetworks]

-def get_face_restorers():
-    return [{"name":x.name(), "cmd_dir": getattr(x, "cmd_dir", None)} for x in shared.face_restorers]
+def get_detailers():
+    return [{"name":x.name(), "cmd_dir": getattr(x, "cmd_dir", None)} for x in shared.detailers]

 def get_prompt_styles():
    return [{ 'name': v.name, 'prompt': v.prompt, 'negative_prompt': v.negative_prompt, 'extra': v.extra, 'filename': v.filename, 'preview': v.preview} for v in shared.prompt_styles.styles.values()]
@@ -91,13 +91,13 @@ def post_interrogate(req: models.ReqInterrogate):
        if req.model not in get_clip_models():
            raise HTTPException(status_code=404, detail="Model not found")
        try:
-            caption = interrogate_image(image, model=req.model, mode=req.mode)
+            caption = interrogate_image(image, clip_model=req.clip_model, blip_model=req.blip_model, mode=req.mode)
        except Exception as e:
            caption = str(e)
        if not req.analyze:
            return models.ResInterrogate(caption=caption)
        else:
-            medium, artist, movement, trending, flavor = analyze_image(image, model=req.model)
+            medium, artist, movement, trending, flavor = analyze_image(image, clip_model=req.clip_model, blip_model=req.blip_model)
            return models.ResInterrogate(caption=caption, medium=medium, artist=artist, movement=movement, trending=trending, flavor=flavor)

 def post_vqa(req: models.ReqVQA):
@@ -158,3 +158,10 @@ def post_pnginfo(req: models.ReqImageInfo):
    params = infotext.parse(geninfo)
    script_callbacks.infotext_pasted_callback(geninfo, params)
    return models.ResImageInfo(info=geninfo, items=items, parameters=params)
+
+def get_history():
+    return shared.history.list
+
+def post_history(req: models.ReqHistory):
+    shared.history.index = shared.history.find(req.name)
+    return shared.history.index
@@ -110,7 +110,7 @@ class ItemHypernetwork(BaseModel):
    name: str = Field(title="Name")
    path: Optional[str] = Field(title="Path")

-class ItemFaceRestorer(BaseModel):
+class ItemDetailer(BaseModel):
    name: str = Field(title="Name")
    cmd_dir: Optional[str] = Field(title="Path")

@@ -259,8 +259,8 @@ class ReqProcess(BaseModel):
    upscaling_resize_h: int = Field(default=512, title="Target Height", ge=1, description="Target height for the upscaler to hit. Only used when resize_mode=1.")
    upscaling_crop: bool = Field(default=True, title="Crop to fit", description="Should the upscaler crop the image to fit in the chosen size?")
    upscaler_1: str = Field(default="None", title="Main upscaler", description=f"The name of the main upscaler to use, it has to be one of this list: {' , '.join([x.name for x in shared.sd_upscalers])}")
-    upscaler_2: str = Field(default="None", title="Secondary upscaler", description=f"The name of the secondary upscaler to use, it has to be one of this list: {' , '.join([x.name for x in shared.sd_upscalers])}")
-    extras_upscaler_2_visibility: float = Field(default=0, title="Secondary upscaler visibility", ge=0, le=1, allow_inf_nan=False, description="Sets the visibility of secondary upscaler, values should be between 0 and 1.")
+    upscaler_2: str = Field(default="None", title="Refine upscaler", description=f"The name of the secondary upscaler to use, it has to be one of this list: {' , '.join([x.name for x in shared.sd_upscalers])}")
+    extras_upscaler_2_visibility: float = Field(default=0, title="Refine upscaler visibility", ge=0, le=1, allow_inf_nan=False, description="Sets the visibility of secondary upscaler, values should be between 0 and 1.")
    upscale_first: bool = Field(default=False, title="Upscale first", description="Should the upscaler run before restoring faces?")

 class ResProcess(BaseModel):
@@ -302,7 +302,8 @@ class ResProgress(BaseModel):

 class ReqInterrogate(BaseModel):
    image: str = Field(default="", title="Image", description="Image to work on, must be a Base64 string containing the image's data.")
-    model: str = Field(default="clip", title="Model", description="The interrogate model used.")
+    clip_model: str = Field(default="", title="CLiP Model", description="The interrogate model used.")
+    blip_model: str = Field(default="", title="BLiP Model", description="The interrogate model used.")

 class ResInterrogate(BaseModel):
    caption: Optional[str] = Field(default=None, title="Caption", description="The generated caption for the image.")
@@ -317,6 +318,9 @@ class ReqVQA(BaseModel):
    model: str = Field(default="MS Florence 2 Base", title="Model", description="The interrogate model used.")
    question: str = Field(default="describe the image", title="Question", description="Question to ask the model.")

+class ReqHistory(BaseModel):
+    name: str = Field(title="Name", description="Name of the history item to select")
+
 class ResVQA(BaseModel):
    answer: Optional[str] = Field(default=None, title="Answer", description="The generated answer for the image.")

@@ -107,7 +107,7 @@ class APIProcess():
        return ResMask(mask=image)

    def post_face(self, req: ReqFace):
-        from scripts.face_details import yolo # pylint: disable=no-name-in-module
+        from shared import yolo # pylint: disable=no-name-in-module
        image = decode_base64_to_image(req.image)
        shared.state.begin('API-FACE', api=True)
        images = []
@@ -59,6 +59,7 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False, name=None):
            else:
                res = list(res)
            if shared.cmd_opts.profile:
+                pr.disable()
                errors.profile(pr, 'Wrap')
        except Exception as e:
            errors.display(e, 'gradio call')
@@ -155,7 +155,7 @@ class Processor():
            for k, v in from_config.items():
                self.load_config[k] = v

-    def load(self, processor_id: str = None) -> str:
+    def load(self, processor_id: str = None, force: bool = True) -> str:
        try:
            t0 = time.time()
            processor_id = processor_id or self.processor_id
@@ -165,6 +165,10 @@ class Processor():
            if self.processor_id != processor_id:
                self.reset()
                self.config(processor_id)
+            else:
+                if not force and self.model is not None:
+                    log.debug(f'Control Processor: id={processor_id} already loaded')
+                    return ''
            if processor_id not in config:
                log.error(f'Control Processor unknown: id="{processor_id}" available={list(config)}')
                return f'Processor failed to load: {processor_id}'
@@ -53,13 +53,14 @@ def control_set(kwargs):
        p_extra_args[k] = v


-def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], inits: List[Image.Image] = [], mask: Image.Image = None, unit_type: str = None, is_generator: bool = True,
+def control_run(state: str = '',
+                units: List[unit.Unit] = [], inputs: List[Image.Image] = [], inits: List[Image.Image] = [], mask: Image.Image = None, unit_type: str = None, is_generator: bool = True,
                input_type: int = 0,
                prompt: str = '', negative_prompt: str = '', styles: List[str] = [],
                steps: int = 20, sampler_index: int = None,
                seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1,
                cfg_scale: float = 6.0, clip_skip: float = 1.0, image_cfg_scale: float = 6.0, diffusers_guidance_rescale: float = 0.7, pag_scale: float = 0.0, pag_adaptive: float = 0.5, cfg_end: float = 1.0,
-                full_quality: bool = True, restore_faces: bool = False, tiling: bool = False, hidiffusion: bool = False,
+                full_quality: bool = True, detailer: bool = False, tiling: bool = False, hidiffusion: bool = False,
                hdr_mode: int = 0, hdr_brightness: float = 0, hdr_color: float = 0, hdr_sharpen: float = 0, hdr_clamp: bool = False, hdr_boundary: float = 4.0, hdr_threshold: float = 0.95,
                hdr_maximize: bool = False, hdr_max_center: float = 0.6, hdr_max_boundry: float = 1.0, hdr_color_picker: str = None, hdr_tint_ratio: float = 0,
                resize_mode_before: int = 0, resize_name_before: str = 'None', resize_context_before: str = 'None', width_before: int = 512, height_before: int = 512, scale_by_before: float = 1.0, selected_scale_tab_before: int = 0,
@@ -71,6 +72,20 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
                video_skip_frames: int = 0, video_type: str = 'None', video_duration: float = 2.0, video_loop: bool = False, video_pad: int = 0, video_interpolate: int = 0,
                *input_script_args
        ):
+    # handle optional initialization via ui
+    for u in units:
+        if not u.enabled:
+            continue
+        if u.process_name is not None and u.process_name != '' and u.process_name != 'None':
+            u.process.load(u.process_name, force=False)
+        if u.model_name is not None and u.model_name != '' and u.model_name != 'None':
+            if u.type == 't2i adapter':
+                u.adapter.load(u.model_name, force=False)
+            else:
+                u.controlnet.load(u.model_name, force=False)
+        if u.process is not None and u.process.override is None and u.override is not None:
+            u.process.override = u.override
+
    global instance, pipe, original_pipeline # pylint: disable=global-statement
    t_start = time.time()
    debug(f'Control: type={unit_type} input={inputs} init={inits} type={input_type}')
@@ -114,7 +129,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
        pag_scale = pag_scale,
        pag_adaptive = pag_adaptive,
        full_quality = full_quality,
-        restore_faces = restore_faces,
+        detailer = detailer,
        tiling = tiling,
        hidiffusion = hidiffusion,
        # resize
@@ -134,6 +149,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
        outpath_samples=shared.opts.outdir_samples or shared.opts.outdir_control_samples,
        outpath_grids=shared.opts.outdir_grids or shared.opts.outdir_control_grids,
    )
+    p.state = state
    # processing.process_init(p)
    resize_mode_before = resize_mode_before if resize_name_before != 'None' and inputs is not None and len(inputs) > 0 else 0

@@ -548,7 +564,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
                            return [], '', '', 'Reference mode without image'
                    elif unit_type == 'controlnet' and has_models:
                        if input_type == 0: # Control only
-                            if shared.sd_model_type == 'f1':
+                            if shared.sd_model_type == 'f1' and 'control_image' not in p.task_args:
                                p.task_args['control_image'] = p.init_images # flux controlnet mandates this
                                p.task_args['strength'] = p.denoising_strength
                        elif input_type == 1: # Init image same as control
@@ -714,15 +730,17 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
    if video_type != 'None' and isinstance(output_images, list):
        p.do_not_save_grid = True # pylint: disable=attribute-defined-outside-init
        output_filename = images.save_video(p, filename=None, images=output_images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate, sync=True)
+        if shared.opts.gradio_skip_video:
+            output_filename = ''
        image_txt = f'| Frames {len(output_images)} | Size {output_images[0].width}x{output_images[0].height}'

+    p.close()
    restore_pipeline()
    debug(f'Ready: {image_txt}')

    html_txt = f'<p>Ready {image_txt}</p>'
    if len(info_txt) > 0:
        html_txt = html_txt + infotext_to_html(info_txt[0])
-
    if is_generator:
        yield (output_images, blended_image, html_txt, output_filename)
    else:
@@ -18,6 +18,7 @@ unit_types = ['t2i adapter', 'controlnet', 'xs', 'lite', 'reference', 'ip']
 class Unit(): # mashup of gradio controls and mapping to actual implementation classes
    def __init__(self,
                 # values
+                 index: int = None,
                 enabled: bool = None,
                 strength: float = None,
                 unit_type: str = None,
@@ -40,15 +41,20 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
                 result_txt = None,
                 extra_controls: list = [],
        ):
+        self.controls = [gr.Label(value=unit_type, visible=False)] # separator
+        self.index = index
        self.enabled = enabled or False
        self.type = unit_type
        self.strength = strength or 1.0
+        self.model_strength = model_strength
        self.start = start or 0
        self.end = end or 1
        self.start = min(self.start, self.end)
        self.end = max(self.start, self.end)
        self.mode = None
        # processor always exists, adapter and controlnet are optional
+        self.model_name = None
+        self.process_name = None
        self.process: processors.Processor = processors.Processor()
        self.adapter: t2iadapter.Adapter = None
        self.controlnet: Union[controlnet.ControlNet, xs.ControlNetXS] = None
@@ -155,6 +161,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
                if isinstance(model_id, str):
                    self.adapter.load(model_id)
                else:
+                    self.controls.append(model_id)
                    model_id.change(fn=self.adapter.load, inputs=[model_id], outputs=[result_txt], show_progress=True)
            if extra_controls is not None and len(extra_controls) > 0:
                extra_controls[0].change(fn=adapter_extra, inputs=extra_controls)
@@ -163,6 +170,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
                if isinstance(model_id, str):
                    self.controlnet.load(model_id)
                else:
+                    self.controls.append(model_id)
                    model_id.change(fn=self.controlnet.load, inputs=[model_id], outputs=[result_txt], show_progress=True)
                    model_id.change(fn=control_mode_show, inputs=[model_id], outputs=[control_mode], show_progress=False)
            if extra_controls is not None and len(extra_controls) > 0:
@@ -172,6 +180,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
                if isinstance(model_id, str):
                    self.controlnet.load(model_id)
                else:
+                    self.controls.append(model_id)
                    model_id.change(fn=self.controlnet.load, inputs=[model_id, extra_controls[0]], outputs=[result_txt], show_progress=True)
            if extra_controls is not None and len(extra_controls) > 0:
                extra_controls[0].change(fn=controlnetxs_extra, inputs=extra_controls)
@@ -180,6 +189,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
                if isinstance(model_id, str):
                    self.controlnet.load(model_id)
                else:
+                    self.controls.append(model_id)
                    model_id.change(fn=self.controlnet.load, inputs=[model_id], outputs=[result_txt], show_progress=True)
            if extra_controls is not None and len(extra_controls) > 0:
                extra_controls[0].change(fn=controlnetxs_extra, inputs=extra_controls)
@@ -189,14 +199,18 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
                extra_controls[1].change(fn=reference_extra, inputs=extra_controls)
                extra_controls[2].change(fn=reference_extra, inputs=extra_controls)
                extra_controls[3].change(fn=reference_extra, inputs=extra_controls)
+
        if enabled_cb is not None:
+            self.controls.append(enabled_cb)
            enabled_cb.change(fn=enabled_change, inputs=[enabled_cb])
        if model_strength is not None:
+            self.controls.append(model_strength)
            model_strength.change(fn=strength_change, inputs=[model_strength])
        if process_id is not None:
            if isinstance(process_id, str):
                self.process.load(process_id)
            else:
+                self.controls.append(process_id)
                process_id.change(fn=self.process.load, inputs=[process_id], outputs=[result_txt], show_progress=True)
        if reset_btn is not None:
            reset_btn.click(fn=reset, inputs=[], outputs=[enabled_cb, model_id, process_id, model_strength])
@@ -207,9 +221,13 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
        if image_reuse is not None:
            image_reuse.click(fn=reuse_image, inputs=[preview_process], outputs=[image_preview]) # return list of images for gallery
        if image_preview is not None:
+            self.controls.append(image_preview)
            image_preview.change(fn=set_image, inputs=[image_preview], outputs=[image_preview])
        if control_start is not None and control_end is not None:
+            self.controls.append(control_start)
+            self.controls.append(control_end)
            control_start.change(fn=control_change, inputs=[control_start, control_end])
            control_end.change(fn=control_change, inputs=[control_start, control_end])
        if control_mode is not None:
+            self.controls.append(control_mode)
            control_mode.change(fn=control_mode_change, inputs=[control_mode])
@@ -4,7 +4,7 @@ from typing import Union
 from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, FluxPipeline, ControlNetModel
 from modules.control.units import detect
 from modules.shared import log, opts, listdir
-from modules import errors, sd_models, devices
+from modules import errors, sd_models, devices, model_quant


 what = 'ControlNet'
@@ -69,12 +69,15 @@ predefined_sdxl = {
 predefined_f1 = {
    "InstantX Union": 'InstantX/FLUX.1-dev-Controlnet-Union',
    "InstantX Canny": 'InstantX/FLUX.1-dev-Controlnet-Canny',
+    "JasperAI Depth": 'jasperai/Flux.1-dev-Controlnet-Depth',
+    "JasperAI Surface Normals": 'jasperai/Flux.1-dev-Controlnet-Surface-Normals',
+    "JasperAI Upscaler": 'jasperai/Flux.1-dev-Controlnet-Upscaler',
    "Shakker-Labs Union": 'Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro',
    "Shakker-Labs Pose": 'Shakker-Labs/FLUX.1-dev-ControlNet-Pose',
    "Shakker-Labs Depth": 'Shakker-Labs/FLUX.1-dev-ControlNet-Depth',
-    "XLabs-AI Canny": 'XLabs-AI/flux-controlnet-canny-v3',
-    "XLabs-AI Depth": 'XLabs-AI/flux-controlnet-depth-v3',
-    "XLabs-AI HED": 'XLabs-AI/flux-controlnet-hed-v3',
+    "XLabs-AI Canny": 'XLabs-AI/flux-controlnet-canny-diffusers',
+    "XLabs-AI Depth": 'XLabs-AI/flux-controlnet-depth-diffusers',
+    "XLabs-AI HED": 'XLabs-AI/flux-controlnet-hed-diffusers'
 }
 models = {}
 all_models = {}
@@ -181,7 +184,7 @@ class ControlNet():
        cls = self.get_class()
        self.model = cls.from_single_file(model_path, **self.load_config)

-    def load(self, model_id: str = None) -> str:
+    def load(self, model_id: str = None, force: bool = True) -> str:
        try:
            t0 = time.time()
            model_id = model_id or self.model_id
@@ -197,6 +200,9 @@ class ControlNet():
            if model_path is None:
                log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id')
                return
+            if model_id == self.model_id and not force:
+                log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded')
+                return
            log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}"')
            if model_path.endswith('.safetensors'):
                self.load_safetensors(model_path)
@@ -205,6 +211,9 @@ class ControlNet():
                    model_path = model_path.replace('/bin', '')
                    self.load_config['use_safetensors'] = False
                cls = self.get_class()
+                if cls is None:
+                    log.error(f'Control {what} model load failed: id="{model_id}" unknown base model')
+                    return
                self.model = cls.from_pretrained(model_path, **self.load_config)
            if self.dtype is not None:
                self.model.to(self.dtype)
@@ -220,8 +229,7 @@ class ControlNet():
            elif "ControlNet" in opts.optimum_quanto_weights:
                try:
                    log.debug(f'Control {what} model Optimum Quanto: id="{model_id}"')
-                    from installer import install
-                    install('optimum-quanto', quiet=True)
+                    model_quant.load_quanto('Load model: type=ControlNet')
                    from modules.sd_models_compile import optimum_quanto_model
                    self.model = optimum_quanto_model(self.model)
                except Exception as e:
@@ -276,7 +284,7 @@ class ControlNetPipeline():
        elif detect.is_f1(pipeline):
            from diffusers import FluxControlNetPipeline
            self.pipeline = FluxControlNetPipeline(
-                vae=pipeline.vae,
+                vae=pipeline.vae.to(devices.device),
                text_encoder=pipeline.text_encoder,
                text_encoder_2=pipeline.text_encoder_2,
                tokenizer=pipeline.tokenizer,
@@ -78,7 +78,7 @@ class ControlLLLite():
        self.model = None
        self.model_id = None

-    def load(self, model_id: str = None) -> str:
+    def load(self, model_id: str = None, force: bool = True) -> str:
        try:
            t0 = time.time()
            model_id = model_id or self.model_id
@@ -94,6 +94,9 @@ class ControlLLLite():
            if model_path is None:
                log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id')
                return
+            if model_id == self.model_id and not force:
+                log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded')
+                return
            log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}" {self.load_config}')
            if model_path.endswith('.safetensors'):
                self.model = ControlNetLLLite(model_path)
@@ -86,7 +86,7 @@ class Adapter():
        self.model = None
        self.model_id = None

-    def load(self, model_id: str = None) -> str:
+    def load(self, model_id: str = None, force: bool = True) -> str:
        try:
            t0 = time.time()
            model_id = model_id or self.model_id
@@ -100,6 +100,9 @@ class Adapter():
            if model_path is None:
                log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id')
                return
+            if model_id == self.model_id and not force:
+                log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded')
+                return
            log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}"')
            if model_path.endswith('.pth') or model_path.endswith('.pt') or model_path.endswith('.safetensors') or model_path.endswith('.bin'):
                from huggingface_hub import hf_hub_download
@@ -74,7 +74,7 @@ class ControlNetXS():
        self.model = None
        self.model_id = None

-    def load(self, model_id: str = None, time_embedding_mix: float = 0.0) -> str:
+    def load(self, model_id: str = None, time_embedding_mix: float = 0.0, force: bool = True) -> str:
        try:
            t0 = time.time()
            model_id = model_id or self.model_id
@@ -90,6 +90,9 @@ class ControlNetXS():
            if model_path is None:
                log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id')
                return
+            if model_id == self.model_id and not force:
+                log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded')
+                return
            self.load_config['time_embedding_mix'] = time_embedding_mix
            log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}" {self.load_config}')
            if model_path.endswith('.safetensors'):
@@ -35,7 +35,7 @@ def HWC3(x):


 def make_noise_disk(H, W, C, F):
-    noise = np.random.uniform(low=0, high=1, size=((H // F) + 2, (W // F) + 2, C)) # noqa
+    noise = np.random.uniform(low=0, high=1, size=((H // F) + 2, (W // F) + 2, C))
    noise = cv2.resize(noise, (W + 2 * F, H + 2 * F), interpolation=cv2.INTER_CUBIC)
    noise = noise[F: F + H, F: F + W]
    noise -= np.min(noise)
@@ -0,0 +1,658 @@
+from copy import deepcopy
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from diffusers import StableDiffusionXLPipeline
+from diffusers.image_processor import PipelineImageInput
+from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_img2img import rescale_noise_cfg, retrieve_latents, retrieve_timesteps
+from diffusers.utils import BaseOutput, deprecate
+from diffusers.utils.torch_utils import randn_tensor
+import numpy as np
+import PIL
+import torch
+from .sdxl import register_attr
+from .media import preprocess
+from .utils import batch_dict_to_tensor, batch_tensor_to_dict, noise_prev, noise_t2t
+
+
+BATCH_ORDER = [
+    "structure_uncond", "appearance_uncond", "uncond", "structure_cond", "appearance_cond", "cond",
+]
+
+
+def get_last_control_i(control_schedule, num_inference_steps):
+    if control_schedule is None:
+        return num_inference_steps, num_inference_steps
+
+    def max_(l):
+        if len(l) == 0:
+            return 0.0
+        return max(l)
+
+    structure_max = 0.0
+    appearance_max = 0.0
+    for block in control_schedule.values():
+        if isinstance(block, list):  # Handling mid_block
+            block = {0: block}
+        for layer in block.values():
+            structure_max = max(structure_max, max_(layer[0] + layer[1]))
+            appearance_max = max(appearance_max, max_(layer[2]))
+
+    structure_i = round(num_inference_steps * structure_max)
+    appearance_i = round(num_inference_steps * appearance_max)
+    return structure_i, appearance_i
+
+
+@dataclass
+class CtrlXStableDiffusionXLPipelineOutput(BaseOutput):
+    images: Union[List[PIL.Image.Image], np.ndarray] = None
+    structures: Union[List[PIL.Image.Image], np.ndarray] = None
+    appearances: Union[List[PIL.Image.Image], np.ndarray] = None
+
+
+class CtrlXStableDiffusionXLPipeline(StableDiffusionXLPipeline):  # diffusers==0.28.0
+
+    def prepare_latents(
+        self, image, batch_size, num_images_per_prompt, num_channels_latents, height, width,
+        dtype, device, generator=None, noise=None,
+    ):
+        batch_size = batch_size * num_images_per_prompt
+        if noise is None:
+            shape = (
+                batch_size,
+                num_channels_latents,
+                height // self.vae_scale_factor,
+                width // self.vae_scale_factor
+            )
+            noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
+            noise = noise * self.scheduler.init_noise_sigma  # Starting noise, need to scale
+        else:
+            noise = noise.to(device)
+
+        if image is None:
+            return noise, None
+
+        if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)):
+            raise ValueError(
+                f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}"
+            )
+
+        # Offload text encoder if `enable_model_cpu_offload` was enabled
+        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:
+            self.text_encoder_2.to("cpu")
+            torch.cuda.empty_cache()
+
+        image = image.to(device=device, dtype=dtype)
+
+        if image.shape[1] == 4:  # Image already in latents form
+            init_latents = image
+
+        else:
+            # Make sure the VAE is in float32 mode, as it overflows in float16
+            if self.vae.config.force_upcast:
+                image = image.to(torch.float32)
+                self.vae.to(torch.float32)
+
+            if isinstance(generator, list) and len(generator) != batch_size:
+                raise ValueError(
+                    f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
+                    f" size of {batch_size}. Make sure the batch size matches the length of the generators."
+                )
+            elif isinstance(generator, list):
+                init_latents = [
+                    retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i])
+                    for i in range(batch_size)
+                ]
+                init_latents = torch.cat(init_latents, dim=0)
+            else:
+                init_latents = retrieve_latents(self.vae.encode(image), generator=generator)
+
+            if self.vae.config.force_upcast:
+                self.vae.to(dtype)
+
+            init_latents = init_latents.to(dtype)
+            init_latents = self.vae.config.scaling_factor * init_latents
+
+        if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0:
+            # Expand init_latents for batch_size
+            additional_image_per_prompt = batch_size // init_latents.shape[0]
+            init_latents = torch.cat([init_latents] * additional_image_per_prompt, dim=0)
+        elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0:
+            raise ValueError(
+                f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts."
+            )
+        else:
+            init_latents = torch.cat([init_latents], dim=0)
+
+        return noise, init_latents
+
+    @property
+    def structure_guidance_scale(self):
+        return self._guidance_scale if self._structure_guidance_scale is None else self._structure_guidance_scale
+
+    @property
+    def appearance_guidance_scale(self):
+        return self._guidance_scale if self._appearance_guidance_scale is None else self._appearance_guidance_scale
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        prompt: Union[str, List[str]] = None,  # TODO: Support prompt_2 and negative_prompt_2
+        structure_prompt: Optional[Union[str, List[str]]] = None,
+        appearance_prompt: Optional[Union[str, List[str]]] = None,
+        structure_image: Optional[PipelineImageInput] = None,
+        appearance_image: Optional[PipelineImageInput] = None,
+        num_inference_steps: int = 50,
+        timesteps: List[int] = None,
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        positive_prompt: Optional[Union[str, List[str]]] = None,
+        height: Optional[int] = None,
+        width: Optional[int] = None,
+        guidance_scale: float = 5.0,
+        structure_guidance_scale: Optional[float] = None,
+        appearance_guidance_scale: Optional[float] = None,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: float = 0.0,
+        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+        latents: Optional[torch.Tensor] = None,
+        structure_latents: Optional[torch.Tensor] = None,
+        appearance_latents: Optional[torch.Tensor] = None,
+        prompt_embeds: Optional[torch.Tensor] = None,  # Positive prompt is concatenated with prompt, so no embeddings
+        structure_prompt_embeds: Optional[torch.Tensor] = None,
+        appearance_prompt_embeds: Optional[torch.Tensor] = None,
+        negative_prompt_embeds: Optional[torch.Tensor] = None,
+        pooled_prompt_embeds: Optional[torch.Tensor] = None,
+        structure_pooled_prompt_embeds: Optional[torch.Tensor] = None,
+        appearance_pooled_prompt_embeds: Optional[torch.Tensor] = None,
+        negative_pooled_prompt_embeds: Optional[torch.Tensor] = None,
+        control_schedule: Optional[Dict] = None,
+        self_recurrence_schedule: Optional[List[int]] = [],  # Format: [(start, end, num_repeat)]
+        decode_structure: Optional[bool] = True,
+        decode_appearance: Optional[bool] = True,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        cross_attention_kwargs: Optional[Dict[str, Any]] = None,
+        guidance_rescale: float = 0.0,
+        original_size: Tuple[int, int] = None,
+        crops_coords_top_left: Tuple[int, int] = (0, 0),
+        target_size: Tuple[int, int] = None,
+        clip_skip: Optional[int] = None,
+        callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
+        callback_on_step_end_tensor_inputs: List[str] = ["latents"],
+        **kwargs,
+    ):
+        # TODO: Add function argument documentation
+
+        callback = kwargs.pop("callback", None)
+        callback_steps = kwargs.pop("callback_steps", None)
+
+        if callback is not None:
+            deprecate(
+                "callback",
+                "1.0.0",
+                "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
+            )
+        if callback_steps is not None:
+            deprecate(
+                "callback_steps",
+                "1.0.0",
+                "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
+            )
+
+        # 0. Default height and width to U-Net
+        height = height or self.default_sample_size * self.vae_scale_factor
+        width = width or self.default_sample_size * self.vae_scale_factor
+        original_size = original_size or (height, width)
+        target_size = target_size or (height, width)
+
+        # 1. Check inputs. Raise error if not correct
+        self.check_inputs(  # TODO: Custom check_inputs for our method
+            prompt,
+            None,  # prompt_2
+            height,
+            width,
+            callback_steps,
+            negative_prompt = negative_prompt,
+            negative_prompt_2 = None,  # negative_prompt_2
+            prompt_embeds = prompt_embeds,
+            negative_prompt_embeds = negative_prompt_embeds,
+            pooled_prompt_embeds = pooled_prompt_embeds,
+            negative_pooled_prompt_embeds = negative_pooled_prompt_embeds,
+            callback_on_step_end_tensor_inputs = callback_on_step_end_tensor_inputs,
+        )
+
+        self._guidance_scale = guidance_scale
+        self._structure_guidance_scale = structure_guidance_scale
+        self._appearance_guidance_scale = appearance_guidance_scale
+        self._guidance_rescale = guidance_rescale
+        self._clip_skip = clip_skip
+        self._cross_attention_kwargs = cross_attention_kwargs
+        self._denoising_end = None  # denoising_end
+        self._denoising_start = None  # denoising_start
+        self._interrupt = False
+
+        # 2. Define call parameters
+        if prompt is not None and isinstance(prompt, str):
+            batch_size = 1
+        elif prompt is not None and isinstance(prompt, list):
+            batch_size = len(prompt)
+        else:
+            batch_size = prompt_embeds.shape[0]
+
+        if batch_size * num_images_per_prompt != 1:
+            raise ValueError(
+                f"Pipeline currently does not support batch_size={batch_size} and num_images_per_prompt=1. "
+                "Effective batch size (batch_size * num_images_per_prompt) must be 1."
+            )
+
+        device = self._execution_device
+
+        # 3. Encode input prompt
+        text_encoder_lora_scale = (
+            self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
+        )
+
+        if positive_prompt is not None and positive_prompt != "":
+            prompt = prompt + ", " + positive_prompt  # Add positive prompt with comma
+            # By default, only add positive prompt to the appearance prompt and not the structure prompt
+            if appearance_prompt is not None and appearance_prompt != "":
+                appearance_prompt = appearance_prompt + ", " + positive_prompt
+
+        (
+            prompt_embeds_,
+            negative_prompt_embeds,
+            pooled_prompt_embeds_,
+            negative_pooled_prompt_embeds,
+        ) = self.encode_prompt(
+            prompt = prompt,
+            prompt_2 = None,  # prompt_2
+            device = device,
+            num_images_per_prompt = num_images_per_prompt,
+            do_classifier_free_guidance = True,  # self.do_classifier_free_guidance, TODO: Support no CFG
+            negative_prompt = negative_prompt,
+            negative_prompt_2 = None,  # negative_prompt_2
+            prompt_embeds = prompt_embeds,
+            negative_prompt_embeds = negative_prompt_embeds,
+            pooled_prompt_embeds = pooled_prompt_embeds,
+            negative_pooled_prompt_embeds = negative_pooled_prompt_embeds,
+            lora_scale = text_encoder_lora_scale,
+            clip_skip = self.clip_skip,
+        )
+        prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds_], dim=0).to(device)
+        add_text_embeds = torch.cat([negative_pooled_prompt_embeds, pooled_prompt_embeds_], dim=0).to(device)
+
+        # 3.1. Structure prompt embeddings
+        if structure_prompt is not None and structure_prompt != "":
+            (
+                structure_prompt_embeds,
+                negative_structure_prompt_embeds,
+                structure_pooled_prompt_embeds,
+                negative_structure_pooled_prompt_embeds,
+            ) = self.encode_prompt(
+                prompt = structure_prompt,
+                prompt_2 = None,  # prompt_2
+                device = device,
+                num_images_per_prompt = num_images_per_prompt,
+                do_classifier_free_guidance = True,  # self.do_classifier_free_guidance, TODO: Support no CFG
+                negative_prompt = negative_prompt if structure_image is None else "",
+                negative_prompt_2 = None,  # negative_prompt_2
+                prompt_embeds = structure_prompt_embeds,
+                negative_prompt_embeds = None,  # negative_prompt_embeds
+                pooled_prompt_embeds = structure_pooled_prompt_embeds,
+                negative_pooled_prompt_embeds = None,  # negative_pooled_prompt_embeds
+                lora_scale = text_encoder_lora_scale,
+                clip_skip = self.clip_skip,
+            )
+            structure_prompt_embeds = torch.cat(
+                [negative_structure_prompt_embeds, structure_prompt_embeds], dim=0
+            ).to(device)
+            structure_add_text_embeds = torch.cat(
+                [negative_structure_pooled_prompt_embeds, structure_pooled_prompt_embeds], dim=0
+            ).to(device)
+        else:
+            structure_prompt_embeds = prompt_embeds
+            structure_add_text_embeds = add_text_embeds
+
+        # 3.2. Appearance prompt embeddings
+        if appearance_prompt is not None and appearance_prompt != "":
+            (
+                appearance_prompt_embeds,
+                negative_appearance_prompt_embeds,
+                appearance_pooled_prompt_embeds,
+                negative_appearance_pooled_prompt_embeds,
+            ) = self.encode_prompt(
+                prompt = appearance_prompt,
+                prompt_2 = None,  # prompt_2
+                device = device,
+                num_images_per_prompt = num_images_per_prompt,
+                do_classifier_free_guidance = True,  # self.do_classifier_free_guidance, TODO: Support no CFG
+                negative_prompt = negative_prompt if appearance_image is None else "",
+                negative_prompt_2 = None,  # negative_prompt_2
+                prompt_embeds = appearance_prompt_embeds,
+                negative_prompt_embeds = None,  # negative_prompt_embeds
+                pooled_prompt_embeds = appearance_pooled_prompt_embeds,  # pooled_prompt_embeds
+                negative_pooled_prompt_embeds = None,  # negative_pooled_prompt_embeds
+                lora_scale = text_encoder_lora_scale,
+                clip_skip = self.clip_skip,
+            )
+            appearance_prompt_embeds = torch.cat(
+                [negative_appearance_prompt_embeds, appearance_prompt_embeds], dim=0
+            ).to(device)
+            appearance_add_text_embeds = torch.cat(
+                [negative_appearance_pooled_prompt_embeds, appearance_pooled_prompt_embeds], dim=0
+            ).to(device)
+        else:
+            appearance_prompt_embeds = prompt_embeds
+            appearance_add_text_embeds = add_text_embeds
+
+        # 3.3. Prepare added time ids & embeddings, TODO: Support no CFG
+        if self.text_encoder_2 is None:
+            text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
+        else:
+            text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
+
+        add_time_ids = self._get_add_time_ids(
+            original_size,
+            crops_coords_top_left,
+            target_size,
+            dtype = prompt_embeds.dtype,
+            text_encoder_projection_dim = text_encoder_projection_dim,
+        )
+        negative_add_time_ids = add_time_ids
+        add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0).to(device)
+
+        # 4. Prepare timesteps
+        timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps)
+
+        # 5. Prepare latent variables
+        num_channels_latents = self.unet.config.in_channels
+
+        latents, _ = self.prepare_latents(
+            None, batch_size, num_images_per_prompt, num_channels_latents, height, width,
+            prompt_embeds.dtype, device, generator, latents
+        )
+
+        if structure_image is not None:
+            structure_image = preprocess(  # Center crop + resize
+                structure_image, self.image_processor, height=height, width=width, resize_mode="crop"
+            )
+            _, clean_structure_latents = self.prepare_latents(
+                structure_image, batch_size, num_images_per_prompt, num_channels_latents, height, width,
+                prompt_embeds.dtype, device, generator, structure_latents,
+            )
+        else:
+            clean_structure_latents = None
+        structure_latents = latents if structure_latents is None else structure_latents
+
+        if appearance_image is not None:
+            appearance_image = preprocess(  # Center crop + resize
+                appearance_image, self.image_processor, height=height, width=width, resize_mode="crop"
+            )
+            _, clean_appearance_latents = self.prepare_latents(
+                appearance_image, batch_size, num_images_per_prompt, num_channels_latents, height, width,
+                prompt_embeds.dtype, device, generator, appearance_latents,
+            )
+        else:
+            clean_appearance_latents = None
+        appearance_latents = latents if appearance_latents is None else appearance_latents
+
+        # 6. Prepare extra step kwargs
+        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
+
+        # 7. Denoising loop
+        num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
+
+        # 7.1 Apply denoising_end
+        def denoising_value_valid(dnv):
+            return isinstance(self.denoising_end, float) and 0 < dnv < 1
+
+        if (
+            self.denoising_end is not None
+            and self.denoising_start is not None
+            and denoising_value_valid(self.denoising_end)
+            and denoising_value_valid(self.denoising_start)
+            and self.denoising_start >= self.denoising_end
+        ):
+            raise ValueError(f"`denoising_start`: {self.denoising_start} cannot be larger than or equal to `denoising_end`: {self.denoising_end} when using type float.")
+        elif self.denoising_end is not None and denoising_value_valid(self.denoising_end):
+            discrete_timestep_cutoff = int(
+                round(
+                    self.scheduler.config.num_train_timesteps
+                    - (self.denoising_end * self.scheduler.config.num_train_timesteps)
+                )
+            )
+            num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
+            timesteps = timesteps[:num_inference_steps]
+
+        # 7.2 Optionally get guidance scale embedding
+        timestep_cond = None
+        if self.unet.config.time_cond_proj_dim is not None:  # TODO: Make guidance scale embedding work with batch_order
+            guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
+            timestep_cond = self.get_guidance_scale_embedding(
+                guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
+            ).to(device=device, dtype=latents.dtype)
+
+        # 7.3 Get batch order
+        batch_order = deepcopy(BATCH_ORDER)
+        if structure_image is not None:  # If image is provided, not generating, so no CFG needed
+            batch_order.remove("structure_uncond")
+        if appearance_image is not None:
+            batch_order.remove("appearance_uncond")
+
+        structure_control_stop_i, appearance_control_stop_i = get_last_control_i(control_schedule, num_inference_steps)
+        if self_recurrence_schedule is None or len(self_recurrence_schedule) == 0:
+            self_recurrence_schedule = [0] * num_inference_steps
+
+        self._num_timesteps = len(timesteps)
+        with self.progress_bar(total=num_inference_steps) as progress_bar:
+            for i, t in enumerate(timesteps):
+                if self.interrupt:
+                    continue
+
+                if i == structure_control_stop_i:  # If not generating structure/appearance, drop after last control
+                    if "structure_uncond" not in batch_order:
+                        batch_order.remove("structure_cond")
+                if i == appearance_control_stop_i:
+                    if "appearance_uncond" not in batch_order:
+                        batch_order.remove("appearance_cond")
+
+                register_attr(self, t=t.item(), do_control=True, batch_order=batch_order)
+
+                # TODO: For now, assume we are doing classifier-free guidance, support no CF-guidance later
+                latent_model_input = self.scheduler.scale_model_input(latents, t)
+                structure_latent_model_input = self.scheduler.scale_model_input(structure_latents, t)
+                appearance_latent_model_input = self.scheduler.scale_model_input(appearance_latents, t)
+
+                all_latent_model_input = {
+                    "structure_uncond": structure_latent_model_input[0:1],
+                    "appearance_uncond": appearance_latent_model_input[0:1],
+                    "uncond": latent_model_input[0:1],
+                    "structure_cond": structure_latent_model_input[0:1],
+                    "appearance_cond": appearance_latent_model_input[0:1],
+                    "cond": latent_model_input[0:1],
+                }
+                all_prompt_embeds = {
+                    "structure_uncond": structure_prompt_embeds[0:1],
+                    "appearance_uncond": appearance_prompt_embeds[0:1],
+                    "uncond": prompt_embeds[0:1],
+                    "structure_cond": structure_prompt_embeds[1:2],
+                    "appearance_cond": appearance_prompt_embeds[1:2],
+                    "cond": prompt_embeds[1:2],
+                }
+                all_add_text_embeds = {
+                    "structure_uncond": structure_add_text_embeds[0:1],
+                    "appearance_uncond": appearance_add_text_embeds[0:1],
+                    "uncond": add_text_embeds[0:1],
+                    "structure_cond": structure_add_text_embeds[1:2],
+                    "appearance_cond": appearance_add_text_embeds[1:2],
+                    "cond": add_text_embeds[1:2],
+                }
+                all_time_ids = {
+                    "structure_uncond": add_time_ids[0:1],
+                    "appearance_uncond": add_time_ids[0:1],
+                    "uncond": add_time_ids[0:1],
+                    "structure_cond": add_time_ids[1:2],
+                    "appearance_cond": add_time_ids[1:2],
+                    "cond": add_time_ids[1:2],
+                }
+
+                concat_latent_model_input = batch_dict_to_tensor(all_latent_model_input, batch_order)
+                concat_prompt_embeds = batch_dict_to_tensor(all_prompt_embeds, batch_order)
+                concat_add_text_embeds = batch_dict_to_tensor(all_add_text_embeds, batch_order)
+                concat_add_time_ids = batch_dict_to_tensor(all_time_ids, batch_order)
+
+                # Predict the noise residual
+                added_cond_kwargs = {"text_embeds": concat_add_text_embeds, "time_ids": concat_add_time_ids}
+
+                concat_noise_pred = self.unet(
+                    concat_latent_model_input,
+                    t,
+                    encoder_hidden_states = concat_prompt_embeds,
+                    timestep_cond = timestep_cond,
+                    cross_attention_kwargs = self.cross_attention_kwargs,
+                    added_cond_kwargs = added_cond_kwargs,
+                ).sample
+                all_noise_pred = batch_tensor_to_dict(concat_noise_pred, batch_order)
+
+                # Classifier-free guidance, TODO: Support no CFG
+                noise_pred = all_noise_pred["uncond"] +\
+                    self.guidance_scale * (all_noise_pred["cond"] - all_noise_pred["uncond"])
+
+                structure_noise_pred = all_noise_pred["structure_cond"]\
+                    if "structure_cond" in batch_order else noise_pred
+                if "structure_uncond" in all_noise_pred:
+                    structure_noise_pred = all_noise_pred["structure_uncond"] +\
+                        self.structure_guidance_scale * (structure_noise_pred - all_noise_pred["structure_uncond"])
+
+                appearance_noise_pred = all_noise_pred["appearance_cond"]\
+                    if "appearance_cond" in batch_order else noise_pred
+                if "appearance_uncond" in all_noise_pred:
+                    appearance_noise_pred = all_noise_pred["appearance_uncond"] +\
+                        self.appearance_guidance_scale * (appearance_noise_pred - all_noise_pred["appearance_uncond"])
+
+                if self.guidance_rescale > 0.0:
+                    noise_pred = rescale_noise_cfg(
+                        noise_pred, all_noise_pred["cond"], guidance_rescale=self.guidance_rescale
+                    )
+                    if "structure_uncond" in all_noise_pred:
+                        structure_noise_pred = rescale_noise_cfg(
+                            structure_noise_pred, all_noise_pred["structure_cond"],
+                            guidance_rescale=self.guidance_rescale
+                        )
+                    if "appearance_uncond" in all_noise_pred:
+                        appearance_noise_pred = rescale_noise_cfg(
+                            appearance_noise_pred, all_noise_pred["appearance_cond"],
+                            guidance_rescale=self.guidance_rescale
+                        )
+
+                # Compute the previous noisy sample x_t -> x_t-1
+                concat_noise_pred = torch.cat(
+                    [structure_noise_pred, appearance_noise_pred, noise_pred], dim=0,
+                )
+                concat_latents = torch.cat(
+                    [structure_latents, appearance_latents, latents], dim=0,
+                )
+                structure_latents, appearance_latents, latents = self.scheduler.step(
+                    concat_noise_pred, t, concat_latents, **extra_step_kwargs,
+                ).prev_sample.chunk(3)
+
+                if clean_structure_latents is not None:
+                    structure_latents = noise_prev(self.scheduler, t, clean_structure_latents)
+                if clean_appearance_latents is not None:
+                    appearance_latents = noise_prev(self.scheduler, t, clean_appearance_latents)
+
+                # Self-recurrence
+                for _ in range(self_recurrence_schedule[i]):
+                    if hasattr(self.scheduler, "_step_index"):  # For fancier schedulers
+                        self.scheduler._step_index -= 1  # TODO: Does this actually work?
+
+                    t_prev = 0 if i + 1 >= num_inference_steps else timesteps[i + 1]
+                    latents = noise_t2t(self.scheduler, t_prev, t, latents)
+                    latent_model_input = torch.cat([latents] * 2)
+
+                    register_attr(self, t=t.item(), do_control=False, batch_order=["uncond", "cond"])
+
+                    # Predict the noise residual
+                    added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
+                    noise_pred_uncond, noise_pred_ = self.unet(
+                        latent_model_input,
+                        t,
+                        encoder_hidden_states = prompt_embeds,
+                        timestep_cond = timestep_cond,
+                        cross_attention_kwargs = self.cross_attention_kwargs,
+                        added_cond_kwargs = added_cond_kwargs,
+                    ).sample.chunk(2)
+                    noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_ - noise_pred_uncond)
+
+                    if self.guidance_rescale > 0.0:
+                        noise_pred = rescale_noise_cfg(noise_pred, noise_pred_, guidance_rescale=self.guidance_rescale)
+
+                    latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample
+
+                # Callbacks
+                if callback_on_step_end is not None:
+                    callback_kwargs = {}
+                    for k in callback_on_step_end_tensor_inputs:
+                        callback_kwargs[k] = locals()[k]
+                    callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
+
+                    latents = callback_outputs.pop("latents", latents)
+                    prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
+                    negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
+                    add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)
+                    negative_pooled_prompt_embeds = callback_outputs.pop("negative_pooled_prompt_embeds", negative_pooled_prompt_embeds)
+                    add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
+                    # add_neg_time_ids = callback_outputs.pop("add_neg_time_ids", add_neg_time_ids)
+
+                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
+                    progress_bar.update()
+                    if callback is not None and i % callback_steps == 0:
+                        step_idx = i // getattr(self.scheduler, "order", 1)
+                        callback(step_idx, t, latents)
+
+        # "Reconstruction"
+        if clean_structure_latents is not None:
+            structure_latents = clean_structure_latents
+        if clean_appearance_latents is not None:
+            appearance_latents = clean_appearance_latents
+
+        # For passing important information onto the refiner
+        self.refiner_args = {"latents": latents.detach(), "prompt": prompt, "negative_prompt": negative_prompt}
+
+        if output_type != "latent":
+            # Make sure the VAE is in float32 mode, as it overflows in float16
+            if self.vae.config.force_upcast:
+                self.upcast_vae()
+                vae_dtype = next(iter(self.vae.post_quant_conv.parameters())).dtype
+                latents = latents.to(vae_dtype)
+                structure_latents = structure_latents.to(vae_dtype)
+                appearance_latents = appearance_latents.to(vae_dtype)
+
+            image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]
+            image = self.image_processor.postprocess(image, output_type=output_type)
+            if decode_structure:
+                structure = self.vae.decode(structure_latents / self.vae.config.scaling_factor, return_dict=False)[0]
+                structure = self.image_processor.postprocess(structure, output_type=output_type)
+            else:
+                structure = structure_latents
+            if decode_appearance:
+                appearance = self.vae.decode(appearance_latents / self.vae.config.scaling_factor, return_dict=False)[0]
+                appearance = self.image_processor.postprocess(appearance, output_type=output_type)
+            else:
+                appearance = appearance_latents
+
+            # Cast back to fp16 if needed
+            if self.vae.config.force_upcast:
+                self.vae.to(dtype=torch.float16)
+
+        else:
+            # combined = torch.cat([latents, structure_latents, appearance_latents], dim=0)
+            # return CtrlXStableDiffusionXLPipelineOutput(images=combined)
+            return CtrlXStableDiffusionXLPipelineOutput(images=latents, structures=structure_latents, appearances=appearance_latents)
+
+        # Offload all models
+        self.maybe_free_model_hooks()
+
+        if not return_dict:
+            return (image, structure, appearance)
+
+        return CtrlXStableDiffusionXLPipelineOutput(images=image, structures=structure, appearances=appearance)
@@ -0,0 +1,70 @@
+import torch.nn.functional as F
+from .utils import batch_dict_to_tensor, batch_tensor_to_dict
+
+
+def get_schedule(timesteps, schedule):
+    end = round(len(timesteps) * schedule)
+    timesteps = timesteps[:end]
+    return timesteps
+
+
+def get_elem(l, i, default=0.0):
+    if i >= len(l):
+        return default
+    return l[i]
+
+
+def pad_list(l_1, l_2, pad=0.0):
+    max_len = max(len(l_1), len(l_2))
+    l_1 = l_1 + [pad] * (max_len - len(l_1))
+    l_2 = l_2 + [pad] * (max_len - len(l_2))
+    return l_1, l_2
+
+
+def normalize(x, dim):
+    x_mean = x.mean(dim=dim, keepdim=True)
+    x_std = x.std(dim=dim, keepdim=True)
+    x_normalized = (x - x_mean) / x_std
+    return x_normalized
+
+
+# https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html
+def appearance_mean_std(q_c_normed, k_s_normed, v_s):  # c: content, s: style
+    q_c = q_c_normed  # q_c and k_s must be projected from normalized features
+    k_s = k_s_normed
+    mean = F.scaled_dot_product_attention(q_c, k_s, v_s)  # Use scaled_dot_product_attention for efficiency
+    std = (F.scaled_dot_product_attention(q_c, k_s, v_s.square()) - mean.square()).relu().sqrt()
+
+    return mean, std
+
+
+def feature_injection(features, batch_order):
+    assert features.shape[0] % len(batch_order) == 0
+    features_dict = batch_tensor_to_dict(features, batch_order)
+    features_dict["cond"] = features_dict["structure_cond"]
+    features = batch_dict_to_tensor(features_dict, batch_order)
+    return features
+
+
+def appearance_transfer(features, q_normed, k_normed, batch_order, v=None, reshape_fn=None):
+    assert features.shape[0] % len(batch_order) == 0
+
+    features_dict = batch_tensor_to_dict(features, batch_order)
+    q_normed_dict = batch_tensor_to_dict(q_normed, batch_order)
+    k_normed_dict = batch_tensor_to_dict(k_normed, batch_order)
+    v_dict = features_dict
+    if v is not None:
+        v_dict = batch_tensor_to_dict(v, batch_order)
+
+    mean_cond, std_cond = appearance_mean_std(
+        q_normed_dict["cond"], k_normed_dict["appearance_cond"], v_dict["appearance_cond"],
+    )
+
+    if reshape_fn is not None:
+        mean_cond = reshape_fn(mean_cond)
+        std_cond = reshape_fn(std_cond)
+
+    features_dict["cond"] = std_cond * normalize(features_dict["cond"], dim=-2) + mean_cond
+
+    features = batch_dict_to_tensor(features_dict, batch_order)
+    return features
@@ -0,0 +1,21 @@
+import numpy as np
+import torch
+import torchvision.transforms.functional as vF
+import PIL
+
+
+JPEG_QUALITY = 95
+
+
+def preprocess(image, processor, **kwargs):
+    if isinstance(image, PIL.Image.Image):
+        pass
+    elif isinstance(image, np.ndarray):
+        image = PIL.Image.fromarray(image)
+    elif isinstance(image, torch.Tensor):
+        image = vF.to_pil_image(image)
+    else:
+        raise TypeError(f"Image must be of type PIL.Image, np.ndarray, or torch.Tensor, got {type(image)} instead.")
+
+    image = processor.preprocess(image, **kwargs)
+    return image
@@ -0,0 +1,298 @@
+from types import MethodType
+from typing import Optional
+from diffusers.models.attention_processor import Attention
+import torch
+import torch.nn.functional as F
+from .features import feature_injection, normalize, appearance_transfer, get_elem, get_schedule
+
+
+def get_control_config(structure_schedule, appearance_schedule):
+    s = structure_schedule
+    a = appearance_schedule
+
+    control_config =\
+f"""control_schedule:
+    #       structure_conv   structure_attn   appearance_attn  conv/attn
+    encoder:                                                # (num layers)
+        0: [[             ], [             ], [             ]]  # 2/0
+        1: [[             ], [             ], [{a}, {a}     ]]  # 2/2
+        2: [[             ], [             ], [{a}, {a}     ]]  # 2/2
+    middle: [[            ], [             ], [             ]]  # 2/1
+    decoder:
+        0: [[{s}          ], [{s}, {s}, {s}], [0.0, {a}, {a}]]  # 3/3
+        1: [[             ], [             ], [{a}, {a}     ]]  # 3/3
+        2: [[             ], [             ], [             ]]  # 3/0
+
+control_target:
+    - [output_tensor]  # structure_conv   choices: {{hidden_states, output_tensor}}
+    - [query, key]     # structure_attn   choices: {{query, key, value}}
+    - [before]         # appearance_attn  choices: {{before, value, after}}
+
+self_recurrence_schedule:
+    - [0.1, 0.5, 2]  # format: [start, end, num_recurrence]"""
+
+    return control_config
+
+
+def convolution_forward(  # From <class 'diffusers.models.resnet.ResnetBlock2D'>, forward (diffusers==0.28.0)
+    self,
+    input_tensor: torch.Tensor,
+    temb: torch.Tensor,
+    *args, # pylint: disable=unused-argument
+    **kwargs, # pylint: disable=unused-argument
+) -> torch.Tensor:
+    do_structure_control = self.do_control and self.t in self.structure_schedule
+
+    hidden_states = input_tensor
+
+    hidden_states = self.norm1(hidden_states)
+    hidden_states = self.nonlinearity(hidden_states)
+
+    if self.upsample is not None:
+        # upsample_nearest_nhwc fails with large batch sizes. see https://github.com/huggingface/diffusers/issues/984
+        if hidden_states.shape[0] >= 64:
+            input_tensor = input_tensor.contiguous()
+            hidden_states = hidden_states.contiguous()
+        input_tensor = self.upsample(input_tensor)
+        hidden_states = self.upsample(hidden_states)
+    elif self.downsample is not None:
+        input_tensor = self.downsample(input_tensor)
+        hidden_states = self.downsample(hidden_states)
+
+    hidden_states = self.conv1(hidden_states)
+
+    if self.time_emb_proj is not None:
+        if not self.skip_time_act:
+            temb = self.nonlinearity(temb)
+        temb = self.time_emb_proj(temb)[:, :, None, None]
+
+    if self.time_embedding_norm == "default":
+        if temb is not None:
+            hidden_states = hidden_states + temb
+        hidden_states = self.norm2(hidden_states)
+    elif self.time_embedding_norm == "scale_shift":
+        if temb is None:
+            raise ValueError(
+                f" `temb` should not be None when `time_embedding_norm` is {self.time_embedding_norm}"
+            )
+        time_scale, time_shift = torch.chunk(temb, 2, dim=1)
+        hidden_states = self.norm2(hidden_states)
+        hidden_states = hidden_states * (1 + time_scale) + time_shift
+    else:
+        hidden_states = self.norm2(hidden_states)
+
+    hidden_states = self.nonlinearity(hidden_states)
+
+    hidden_states = self.dropout(hidden_states)
+    hidden_states = self.conv2(hidden_states)
+
+    # Feature injection and AdaIN (hidden_states)
+    if do_structure_control and "hidden_states" in self.structure_target:
+        hidden_states = feature_injection(hidden_states, batch_order=self.batch_order)
+
+    if self.conv_shortcut is not None:
+        input_tensor = self.conv_shortcut(input_tensor)
+
+    output_tensor = (input_tensor + hidden_states) / self.output_scale_factor
+
+    # Feature injection and AdaIN (output_tensor)
+    if do_structure_control and "output_tensor" in self.structure_target:
+        output_tensor = feature_injection(output_tensor, batch_order=self.batch_order)
+
+    return output_tensor
+
+
+class AttnProcessor2_0:  # From <class 'diffusers.models.attention_processor.AttnProcessor2_0'> (diffusers==0.28.0)
+
+    def __init__(self):
+        if not hasattr(F, "scaled_dot_product_attention"):
+            raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
+
+    def __call__( # pylint: disable=keyword-arg-before-vararg
+        self,
+        attn: Attention,
+        hidden_states: torch.FloatTensor,
+        encoder_hidden_states: Optional[torch.FloatTensor] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        temb: Optional[torch.FloatTensor] = None,
+        *args,
+        **kwargs,
+    ) -> torch.FloatTensor:
+        do_structure_control = attn.do_control and attn.t in attn.structure_schedule
+        do_appearance_control = attn.do_control and attn.t in attn.appearance_schedule
+
+        residual = hidden_states
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size, sequence_length, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+
+        if attention_mask is not None:
+            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+            # scaled_dot_product_attention expects attention_mask shape to be
+            # (batch, heads, source_length, target_length)
+            attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        no_encoder_hidden_states = encoder_hidden_states is None
+        if no_encoder_hidden_states:
+            encoder_hidden_states = hidden_states
+        elif attn.norm_cross:
+            encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        if do_appearance_control:  # Assume we only have this for self attention
+            hidden_states_normed = normalize(hidden_states, dim=-2)  # B H D C
+            encoder_hidden_states_normed = normalize(encoder_hidden_states, dim=-2)
+
+            query_normed = attn.to_q(hidden_states_normed)
+            key_normed = attn.to_k(encoder_hidden_states_normed)
+
+            inner_dim = key_normed.shape[-1]
+            head_dim = inner_dim // attn.heads
+            query_normed = query_normed.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+            key_normed = key_normed.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+            # Match query and key injection with structure injection (if injection is happening this layer)
+            if do_structure_control:
+                if "query" in attn.structure_target:
+                    query_normed = feature_injection(query_normed, batch_order=attn.batch_order)
+                if "key" in attn.structure_target:
+                    key_normed = feature_injection(key_normed, batch_order=attn.batch_order)
+
+        # Appearance transfer (before)
+        if do_appearance_control and "before" in attn.appearance_target:
+            hidden_states = hidden_states.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+            hidden_states = appearance_transfer(hidden_states, query_normed, key_normed, batch_order=attn.batch_order)
+            hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+
+            if no_encoder_hidden_states:
+                encoder_hidden_states = hidden_states
+            elif attn.norm_cross:
+                encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        query = attn.to_q(hidden_states)
+
+        key = attn.to_k(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states)
+
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // attn.heads
+
+        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        # Feature injection (query, key, and/or value)
+        if do_structure_control:
+            if "query" in attn.structure_target:
+                query = feature_injection(query, batch_order=attn.batch_order)
+            if "key" in attn.structure_target:
+                key = feature_injection(key, batch_order=attn.batch_order)
+            if "value" in attn.structure_target:
+                value = feature_injection(value, batch_order=attn.batch_order)
+
+        # Appearance transfer (value)
+        if do_appearance_control and "value" in attn.appearance_target:
+            value = appearance_transfer(value, query_normed, key_normed, batch_order=attn.batch_order)
+
+        # The output of sdp = (batch, num_heads, seq_len, head_dim)
+        hidden_states = F.scaled_dot_product_attention(
+            query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
+        )
+
+        # Appearance transfer (after)
+        if do_appearance_control and "after" in attn.appearance_target:
+            hidden_states = appearance_transfer(hidden_states, query_normed, key_normed, batch_order=attn.batch_order)
+
+        hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        hidden_states = hidden_states.to(query.dtype)
+
+        # Linear projection
+        hidden_states = attn.to_out[0](hidden_states, *args)
+        # Dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states
+
+
+def register_control(
+    model,
+    timesteps,
+    control_schedule,  # structure_conv, structure_attn, appearance_attn
+    control_target = [["output_tensor"], ["query", "key"], ["before"]],
+):
+    # Assume timesteps in reverse order (T -> 0)
+    for block_type in ["encoder", "decoder", "middle"]:
+        blocks = {
+            "encoder": model.unet.down_blocks,
+            "decoder": model.unet.up_blocks,
+            "middle": [model.unet.mid_block],
+        }[block_type]
+
+        control_schedule_block = control_schedule[block_type]
+        if block_type == "middle":
+            control_schedule_block = [control_schedule_block]
+
+        for layer in range(len(control_schedule_block)):
+            # Convolution
+            num_blocks = len(blocks[layer].resnets) if hasattr(blocks[layer], "resnets") else 0
+            for block in range(num_blocks):
+                convolution = blocks[layer].resnets[block]
+                convolution.structure_target = control_target[0]
+                convolution.structure_schedule = get_schedule(
+                    timesteps, get_elem(control_schedule_block[layer][0], block)
+                )
+                convolution.forward = MethodType(convolution_forward, convolution)
+
+            # Self-attention
+            num_blocks = len(blocks[layer].attentions) if hasattr(blocks[layer], "attentions") else 0
+            for block in range(num_blocks):
+                for transformer_block in blocks[layer].attentions[block].transformer_blocks:
+                    attention = transformer_block.attn1
+                    attention.structure_target = control_target[1]
+                    attention.structure_schedule = get_schedule(
+                        timesteps, get_elem(control_schedule_block[layer][1], block)
+                    )
+                    attention.appearance_target = control_target[2]
+                    attention.appearance_schedule = get_schedule(
+                        timesteps, get_elem(control_schedule_block[layer][2], block)
+                    )
+                    attention.processor = AttnProcessor2_0()
+
+
+def register_attr(model, t, do_control, batch_order):
+    for layer_type in ["encoder", "decoder", "middle"]:
+        blocks = {"encoder": model.unet.down_blocks, "decoder": model.unet.up_blocks,
+                  "middle": [model.unet.mid_block]}[layer_type]
+        for layer in blocks:
+            # Convolution
+            for module in layer.resnets:
+                module.t = t
+                module.do_control = do_control
+                module.batch_order = batch_order
+            # Self-attention
+            if hasattr(layer, "attentions"):
+                for block in layer.attentions:
+                    for module in block.transformer_blocks:
+                        module.attn1.t = t
+                        module.attn1.do_control = do_control
+                        module.attn1.batch_order = batch_order
@@ -0,0 +1,100 @@
+import random
+from os import environ
+import numpy as np
+import torch
+
+
+JPEG_QUALITY = 100
+
+
+def seed_everything(seed):
+    random.seed(seed)
+    environ["PYTHONHASHSEED"] = str(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+
+def exists(x):
+    return x is not None
+
+
+def get(x, default):
+    if exists(x):
+        return x
+    return default
+
+
+def get_self_recurrence_schedule(schedule, num_inference_steps):
+    self_recurrence_schedule = [0] * num_inference_steps
+    for schedule_current in reversed(schedule):
+        if schedule_current is None or len(schedule_current) == 0:
+            continue
+        [start, end, repeat] = schedule_current
+        start_i = round(num_inference_steps * start)
+        end_i = round(num_inference_steps * end)
+        for i in range(start_i, end_i):
+            self_recurrence_schedule[i] = repeat
+    return self_recurrence_schedule
+
+
+def batch_dict_to_tensor(batch_dict, batch_order):
+    batch_tensor = []
+    for batch_type in batch_order:
+        batch_tensor.append(batch_dict[batch_type])
+    batch_tensor = torch.cat(batch_tensor, dim=0)
+    return batch_tensor
+
+
+def batch_tensor_to_dict(batch_tensor, batch_order):
+    batch_tensor_chunk = batch_tensor.chunk(len(batch_order))
+    batch_dict = {}
+    for i, batch_type in enumerate(batch_order):
+        batch_dict[batch_type] = batch_tensor_chunk[i]
+    return batch_dict
+
+
+def noise_prev(scheduler, timestep, x_0, noise=None):
+    if scheduler.num_inference_steps is None:
+        raise ValueError(
+            "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"
+        )
+
+    if noise is None:
+        noise = torch.randn_like(x_0).to(x_0)
+
+    # From DDIMScheduler step function (hopefully this works)
+    timestep_i = (scheduler.timesteps == timestep).nonzero(as_tuple=True)[0][0].item()
+    if timestep_i + 1 >= scheduler.timesteps.shape[0]:  # We are at t = 0 (ish)
+        return x_0
+    prev_timestep = scheduler.timesteps[timestep_i + 1:timestep_i + 2]  # Make sure t is not 0-dim
+
+    x_t_prev = scheduler.add_noise(x_0, noise, prev_timestep)
+    return x_t_prev
+
+
+def noise_t2t(scheduler, timestep, timestep_target, x_t, noise=None):
+    assert timestep_target >= timestep
+    if noise is None:
+        noise = torch.randn_like(x_t).to(x_t)
+
+    alphas_cumprod = scheduler.alphas_cumprod.to(device=x_t.device, dtype=x_t.dtype)
+
+    timestep = timestep.to(torch.long)
+    timestep_target = timestep_target.to(torch.long)
+
+    alpha_prod_t = alphas_cumprod[timestep]
+    alpha_prod_tt = alphas_cumprod[timestep_target]
+    alpha_prod = alpha_prod_tt / alpha_prod_t
+
+    sqrt_alpha_prod = (alpha_prod ** 0.5).flatten()
+    while len(sqrt_alpha_prod.shape) < len(x_t.shape):
+        sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1)
+
+    sqrt_one_minus_alpha_prod = ((1 - alpha_prod) ** 0.5).flatten()
+    while len(sqrt_one_minus_alpha_prod.shape) < len(x_t.shape):
+        sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1)
+
+    x_tt = sqrt_alpha_prod * x_t + sqrt_one_minus_alpha_prod * noise
+    return x_tt
@@ -16,7 +16,7 @@ class DeepDanbooru:
        if self.model is not None:
            return
        model_path = os.path.join(paths.models_path, "DeepDanbooru")
-        shared.log.debug(f'Loading interrogate model: type=DeepDanbooru folder={model_path}')
+        shared.log.debug(f'Load interrogate model: type=DeepDanbooru folder="{model_path}"')
        files = modelloader.load_models(
            model_path=model_path,
            model_url='https://github.com/AUTOMATIC1111/TorchDeepDanbooru/releases/download/v1/model-resnet_custom_v3.pt',
@@ -0,0 +1,17 @@
+from modules import shared
+
+
+class Detailer: # abstract class used for postprocessing
+    def name(self):
+        return "None"
+
+    def restore(self, np_image):
+        return np_image
+
+
+def detail(np_image, p=None): # postprocesses the image
+    detailers = [x for x in shared.detailers if x.name() == shared.opts.detailer_model or shared.opts.detailer_model is None]
+    if len(detailers) == 0:
+        return np_image
+    detailer = detailers[0]
+    return detailer.restore(np_image, p)
@@ -1,26 +1,69 @@
 import os
-import gc
 import sys
 import time
 import contextlib
 from functools import wraps
 import torch
-from modules.errors import log
-from modules import cmd_args, shared, memstats, errors
-
-if sys.platform == "darwin":
-    from modules import mac_specific # pylint: disable=ungrouped-imports
+from modules.errors import log, display, install as install_traceback
+from installer import install


-previous_oom = 0
 debug = os.environ.get('SD_DEVICE_DEBUG', None) is not None
+install_traceback() # traceback handler
+opts = None # initialized in get_backend to avoid circular import
+args = None # initialized in get_backend to avoid circular import
+cuda_ok = torch.cuda.is_available() or (hasattr(torch, 'xpu') and torch.xpu.is_available())
+inference_context = torch.no_grad
+cpu = torch.device("cpu")
+
+fp16_ok = None # set once by test_fp16
+bf16_ok = None # set once by test_bf16
+
+backend = None # set by get_backend
+device = None # set by get_optimal_device
+dtype = None # set by set_dtype
+dtype_vae = None
+dtype_unet = None
+unet_needs_upcast = False # compatibility item
+onnx = None
+sdpa_original = None
+sdpa_pre_dyanmic_atten = None
+previous_oom = 0 # oom counter
+if debug:
+    log.info(f'Torch build config: {torch.__config__.show()}')
+# set_cuda_sync_mode('block') # none/auto/spin/yield/block


 def has_mps() -> bool:
    if sys.platform != "darwin":
        return False
    else:
-        return mac_specific.has_mps # pylint: disable=used-before-assignment
+        from modules import devices_mac # pylint: disable=ungrouped-imports
+        return devices_mac.has_mps # pylint: disable=used-before-assignment
+
+
+def has_xpu() -> bool:
+    return bool(hasattr(torch, 'xpu') and torch.xpu.is_available())
+
+
+def get_backend(shared_cmd_opts):
+    global args # pylint: disable=global-statement
+    args = shared_cmd_opts
+    if args.use_openvino:
+        name = 'openvino'
+    elif args.use_directml:
+        name = 'directml'
+    elif has_xpu():
+        name = 'ipex'
+    elif torch.cuda.is_available() and torch.version.cuda:
+        name = 'cuda'
+    elif torch.cuda.is_available() and torch.version.hip:
+        name = 'rocm'
+    elif sys.platform == 'darwin':
+        name = 'mps'
+    else:
+        name = 'cpu'
+    return name


 def get_gpu_info():
@@ -44,12 +87,13 @@ def get_gpu_info():

    if not torch.cuda.is_available():
        try:
-            if shared.cmd_opts.use_openvino:
+            if backend == 'openvino':
+                from modules.intel.openvino import get_openvino_device
                return {
                    'device': get_openvino_device(), # pylint: disable=used-before-assignment
                    'openvino': get_package_version("openvino"),
                }
-            elif shared.cmd_opts.use_directml:
+            elif backend == 'directml':
                return {
                    'device': f'{torch.cuda.get_device_name(torch.cuda.current_device())} n={torch.cuda.device_count()}',
                    'directml': get_package_version("torch-directml"),
@@ -60,19 +104,19 @@ def get_gpu_info():
            return {}
    else:
        try:
-            if hasattr(torch, "xpu") and torch.xpu.is_available():
+            if backend == 'ipex':
                return {
                    'device': f'{torch.xpu.get_device_name(torch.xpu.current_device())} n={torch.xpu.device_count()}',
                    'ipex': get_package_version('intel-extension-for-pytorch'),
                }
-            elif torch.version.cuda:
+            elif backend == 'cuda':
                return {
-                    'device': f'{torch.cuda.get_device_name(torch.cuda.current_device())} n={torch.cuda.device_count()} arch={torch.cuda.get_arch_list()[-1]} cap={torch.cuda.get_device_capability(device)}',
+                    'device': f'{torch.cuda.get_device_name(torch.cuda.current_device())} n={torch.cuda.device_count()} arch={torch.cuda.get_arch_list()[-1]} capability={torch.cuda.get_device_capability(device)}',
                    'cuda': torch.version.cuda,
                    'cudnn': torch.backends.cudnn.version(),
                    'driver': get_driver(),
                }
-            elif torch.version.hip:
+            elif backend == 'rocm':
                return {
                    'device': f'{torch.cuda.get_device_name(torch.cuda.current_device())} n={torch.cuda.device_count()}',
                    'hip': torch.version.hip,
@@ -83,7 +127,7 @@ def get_gpu_info():
                }
        except Exception as ex:
            if debug:
-                errors.display(ex, 'Device exception')
+                display(ex, 'Device exception')
            return { 'error': ex }


@@ -95,17 +139,18 @@ def extract_device_id(args, name): # pylint: disable=redefined-outer-name


 def get_cuda_device_string():
+    from modules.shared import cmd_opts
    if backend == 'ipex':
-        if shared.cmd_opts.device_id is not None:
-            return f"xpu:{shared.cmd_opts.device_id}"
+        if cmd_opts.device_id is not None:
+            return f"xpu:{cmd_opts.device_id}"
        return "xpu"
    elif backend == 'directml' and torch.dml.is_available():
-        if shared.cmd_opts.device_id is not None:
-            return f"privateuseone:{shared.cmd_opts.device_id}"
+        if cmd_opts.device_id is not None:
+            return f"privateuseone:{cmd_opts.device_id}"
        return torch.dml.get_device_string(torch.dml.default_device().index)
    else:
-        if shared.cmd_opts.device_id is not None:
-            return f"cuda:{shared.cmd_opts.device_id}"
+        if cmd_opts.device_id is not None:
+            return f"cuda:{cmd_opts.device_id}"
        return "cuda"


@@ -121,14 +166,17 @@ def get_optimal_device():
    return torch.device(get_optimal_device_name())


-def get_device_for(task):
-    if task in shared.cmd_opts.use_cpu:
-        log.debug(f'Forcing CPU for task: {task}')
-        return cpu
+def get_device_for(task): # pylint: disable=unused-argument
+    # if task in cmd_opts.use_cpu:
+    #    log.debug(f'Forcing CPU for task: {task}')
+    #    return cpu
    return get_optimal_device()


 def torch_gc(force=False, fast=False):
+    import gc
+    from modules import timer, memstats
+    from modules.shared import cmd_opts
    t0 = time.time()
    mem = memstats.memory_stats()
    gpu = mem.get('gpu', {})
@@ -140,33 +188,38 @@ def torch_gc(force=False, fast=False):
        used_gpu = round(100 * gpu.get('used', 0) / gpu.get('total', 1)) if gpu.get('total', 1) > 1 else 0
    used_ram = round(100 * ram.get('used', 0) / ram.get('total', 1)) if ram.get('total', 1) > 1 else 0
    global previous_oom # pylint: disable=global-statement
-    threshold = 0 if (shared.cmd_opts.lowvram and not shared.cmd_opts.use_zluda) else shared.opts.torch_gc_threshold
+    threshold = 0 if (cmd_opts.lowvram and not cmd_opts.use_zluda) else opts.torch_gc_threshold
+    collected = 0
    if force or threshold == 0 or used_gpu >= threshold or used_ram >= threshold:
        force = True
    if oom > previous_oom:
        previous_oom = oom
        log.warning(f'GPU out-of-memory error: {mem}')
        force = True
-    if not force:
-        return
-
-    # actual gc
-    collected = gc.collect() if not fast else 0 # python gc
-    if cuda_ok:
-        try:
-            with torch.cuda.device(get_cuda_device_string()):
-                torch.cuda.empty_cache() # cuda gc
-                torch.cuda.ipc_collect()
-        except Exception:
-            pass
+    if force:
+        # actual gc
+        collected = gc.collect() if not fast else 0 # python gc
+        if cuda_ok:
+            try:
+                with torch.cuda.device(get_cuda_device_string()):
+                    torch.cuda.empty_cache() # cuda gc
+                    torch.cuda.ipc_collect()
+            except Exception:
+                pass
    t1 = time.time()
+    if 'gc' not in timer.process.records:
+        timer.process.records['gc'] = 0
+    timer.process.records['gc'] += t1 - t0
+    if not force or collected == 0:
+        return
    mem = memstats.memory_stats()
    saved = round(gpu.get('used', 0) - mem.get('gpu', {}).get('used', 0), 2)
    before = { 'gpu': gpu.get('used', 0), 'ram': ram.get('used', 0) }
    after = { 'gpu': mem.get('gpu', {}).get('used', 0), 'ram': mem.get('ram', {}).get('used', 0), 'retries': mem.get('retries', 0), 'oom': mem.get('oom', 0) }
    utilization = { 'gpu': used_gpu, 'ram': used_ram, 'threshold': threshold }
    results = { 'collected': collected, 'saved': saved }
-    log.debug(f'GC: utilization={utilization} gc={results} before={before} after={after} device={torch.device(get_optimal_device_name())} fn={sys._getframe(1).f_code.co_name} time={round(t1 - t0, 2)}') # pylint: disable=protected-access
+    fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
+    log.debug(f'GC: utilization={utilization} gc={results} before={before} after={after} device={torch.device(get_optimal_device_name())} fn={fn} time={round(t1 - t0, 2)}') # pylint: disable=protected-access


 def set_cuda_sync_mode(mode):
@@ -189,185 +242,198 @@ def set_cuda_sync_mode(mode):


 def test_fp16():
-    if shared.cmd_opts.experimental:
-        if debug:
-            log.debug('Torch FP16 test skip')
-        return True
+    global fp16_ok # pylint: disable=global-statement
+    if fp16_ok is not None:
+        return fp16_ok
+    if sys.platform == "darwin" or backend == 'openvino': # override
+        fp16_ok = False
+        return fp16_ok
    try:
        x = torch.tensor([[1.5,.0,.0,.0]]).to(device=device, dtype=torch.float16)
        layerNorm = torch.nn.LayerNorm(4, eps=0.00001, elementwise_affine=True, dtype=torch.float16, device=device)
-        _y = layerNorm(x)
-        if debug:
-            log.debug('Torch FP16 test pass')
-        return True
+        out = layerNorm(x)
+        if out.dtype != torch.float16:
+            raise RuntimeError('Torch FP16 test: dtype mismatch')
+        if torch.all(torch.isnan(out)).item():
+            raise RuntimeError('Torch FP16 test: NaN')
+        fp16_ok = True
    except Exception as ex:
-        log.warning(f'Torch FP16 test failed: Forcing FP32 operations: {ex}')
-        shared.opts.cuda_dtype = 'FP32'
-        shared.opts.no_half = True
-        shared.opts.no_half_vae = True
-        return False
+        log.warning(f'Torch FP16 test fail: {ex}')
+        fp16_ok = False
+    return fp16_ok


 def test_bf16():
-    if shared.cmd_opts.experimental:
-        if debug:
-            log.debug('Torch BF16 test skip')
-        return True
+    global bf16_ok # pylint: disable=global-statement
+    if bf16_ok is not None:
+        return bf16_ok
+    if sys.platform == "darwin" or backend == 'openvino' or backend == 'directml': # override
+        bf16_ok = False
+        return bf16_ok
    try:
        import torch.nn.functional as F
        image = torch.randn(1, 4, 32, 32).to(device=device, dtype=torch.bfloat16)
-        _out = F.interpolate(image, size=(64, 64), mode="nearest")
-        if debug:
-            log.debug('Torch BF16 test pass')
-        return True
-    except Exception:
-        log.warning('Torch BF16 test failed: Fallback to FP16 operations')
-        return False
+        out = F.interpolate(image, size=(64, 64), mode="nearest")
+        if out.dtype != torch.bfloat16:
+            raise RuntimeError('Torch BF16 test: dtype mismatch')
+        if torch.all(torch.isnan(out)).item():
+            raise RuntimeError('Torch BF16 test: NaN')
+        bf16_ok = True
+    except Exception as ex:
+        log.warning(f'Torch BF16 test fail: {ex}')
+        bf16_ok = False
+    return bf16_ok


-def set_cuda_params():
-    if debug:
-        log.debug(f'Verifying Torch settings: cuda={cuda_ok}')
-    if backend == "ipex":
-        try:
-            torch.xpu.set_fp32_math_mode(mode=torch.xpu.FP32MathMode.TF32)
-        except Exception:
-            pass
+def set_cudnn_params():
    if cuda_ok:
        try:
            torch.backends.cuda.matmul.allow_tf32 = True
            torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = True
            torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = True
+            torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp(True)
        except Exception:
            pass
        if torch.backends.cudnn.is_available():
            try:
-                torch.backends.cudnn.deterministic = shared.opts.cudnn_deterministic
-                torch.use_deterministic_algorithms(shared.opts.cudnn_deterministic)
-                log.debug(f'Torch mode: deterministic={shared.opts.cudnn_deterministic}')
-                if shared.opts.cudnn_deterministic:
+                torch.backends.cudnn.deterministic = opts.cudnn_deterministic
+                torch.use_deterministic_algorithms(opts.cudnn_deterministic)
+                if opts.cudnn_deterministic:
                    os.environ.setdefault('CUBLAS_WORKSPACE_CONFIG', ':4096:8')
                torch.backends.cudnn.benchmark = True
-                if shared.opts.cudnn_benchmark:
+                if opts.cudnn_benchmark:
                    log.debug('Torch cuDNN: enable benchmark')
                    torch.backends.cudnn.benchmark_limit = 0
                torch.backends.cudnn.allow_tf32 = True
            except Exception:
                pass
+
+
+def override_ipex_math():
+    if backend == "ipex":
+        try:
+            torch.xpu.set_fp32_math_mode(mode=torch.xpu.FP32MathMode.TF32)
+        except Exception:
+            pass
+
+
+def set_sdpa_params():
    try:
-        if shared.opts.cross_attention_optimization == "Scaled-Dot-Product":
-            torch.backends.cuda.enable_flash_sdp('Flash attention' in shared.opts.sdp_options)
-            torch.backends.cuda.enable_mem_efficient_sdp('Memory attention' in shared.opts.sdp_options)
-            torch.backends.cuda.enable_math_sdp('Math attention' in shared.opts.sdp_options)
+        if opts.cross_attention_optimization == "Scaled-Dot-Product":
+            torch.backends.cuda.enable_flash_sdp('Flash attention' in opts.sdp_options)
+            torch.backends.cuda.enable_mem_efficient_sdp('Memory attention' in opts.sdp_options)
+            torch.backends.cuda.enable_math_sdp('Math attention' in opts.sdp_options)
+            global sdpa_original # pylint: disable=global-statement
+            if sdpa_original is not None:
+                torch.nn.functional.scaled_dot_product_attention = sdpa_original
+            else:
+                sdpa_original = torch.nn.functional.scaled_dot_product_attention
            if backend == "rocm":
-                if 'Flash attention' in shared.opts.sdp_options:
+                if 'Flash attention' in opts.sdp_options:
                    try:
                        # https://github.com/huggingface/diffusers/discussions/7172
                        from flash_attn import flash_attn_func
-                        backup_sdpa = torch.nn.functional.scaled_dot_product_attention
-                        @wraps(torch.nn.functional.scaled_dot_product_attention)
-                        def sdpa_hijack(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
-                            if query.shape[3] <= 128 and attn_mask is None and query.dtype != torch.float32:
+                        sdpa_pre_flash_atten = torch.nn.functional.scaled_dot_product_attention
+                        @wraps(sdpa_pre_flash_atten)
+                        def sdpa_flash_atten(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
+                            if query.shape[-1] <= 128 and attn_mask is None and query.dtype != torch.float32:
                                return flash_attn_func(q=query.transpose(1, 2), k=key.transpose(1, 2), v=value.transpose(1, 2), dropout_p=dropout_p, causal=is_causal, softmax_scale=scale).transpose(1, 2)
                            else:
-                                return backup_sdpa(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale)
-                        torch.nn.functional.scaled_dot_product_attention = sdpa_hijack
-                        shared.log.debug('ROCm Flash Attention Hijacked')
+                                return sdpa_pre_flash_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale)
+                        torch.nn.functional.scaled_dot_product_attention = sdpa_flash_atten
+                        log.debug('ROCm Flash Attention Hijacked')
                    except Exception as err:
                        log.error(f'ROCm Flash Attention failed: {err}')
-            if 'Dynamic attention' in shared.opts.sdp_options:
-                from modules.sd_hijack_dynamic_atten import sliced_scaled_dot_product_attention
-                torch.nn.functional.scaled_dot_product_attention = sliced_scaled_dot_product_attention
+            if 'Sage attention' in opts.sdp_options:
+                try:
+                    install('sageattention')
+                    from sageattention import sageattn
+                    sdpa_pre_sage_atten = torch.nn.functional.scaled_dot_product_attention
+                    @wraps(sdpa_pre_sage_atten)
+                    def sdpa_sage_atten(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
+                        if query.shape[-1] in {128, 96, 64} and attn_mask is None and query.dtype != torch.float32:
+                            return sageattn(q=query, k=key, v=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale)
+                        else:
+                            return sdpa_pre_sage_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale)
+                    torch.nn.functional.scaled_dot_product_attention = sdpa_sage_atten
+                    log.debug('SDPA Sage Attention Hijacked')
+                except Exception as err:
+                    log.error(f'SDPA Sage Attention failed: {err}')
+            if 'Dynamic attention' in opts.sdp_options:
+                try:
+                    global sdpa_pre_dyanmic_atten # pylint: disable=global-statement
+                    sdpa_pre_dyanmic_atten = torch.nn.functional.scaled_dot_product_attention
+                    from modules.sd_hijack_dynamic_atten import sliced_scaled_dot_product_attention
+                    torch.nn.functional.scaled_dot_product_attention = sliced_scaled_dot_product_attention
+                    log.debug('SDPA Dynamic Attention Hijacked')
+                except Exception as err:
+                    log.error(f'SDPA Dynamic Attention failed: {err}')
    except Exception:
        pass
-    if shared.cmd_opts.profile:
-        shared.log.debug(f'Torch info: {torch.__config__.show()}')
-    global dtype, dtype_vae, dtype_unet, unet_needs_upcast, inference_context, fp16_ok, bf16_ok # pylint: disable=global-statement
-    if shared.opts.cuda_dtype == 'FP32':
+
+
+def set_dtype():
+    global dtype, dtype_vae, dtype_unet, unet_needs_upcast, inference_context # pylint: disable=global-statement
+    test_fp16()
+    test_bf16()
+    if opts.cuda_dtype == 'Auto': # detect
+        if bf16_ok:
+            dtype = torch.bfloat16
+            dtype_vae = torch.bfloat16
+            dtype_unet = torch.bfloat16
+        elif fp16_ok:
+            dtype = torch.float16
+            dtype_vae = torch.float16
+            dtype_unet = torch.float16
+        else:
+            dtype = torch.float32
+            dtype_vae = torch.float32
+            dtype_unet = torch.float32
+    elif opts.cuda_dtype == 'FP32':
        dtype = torch.float32
        dtype_vae = torch.float32
        dtype_unet = torch.float32
-        fp16_ok = None
-        bf16_ok = None
-    elif shared.opts.cuda_dtype == 'BF16' or dtype == torch.bfloat16:
-        fp16_ok = test_fp16() if fp16_ok is None else fp16_ok
-        bf16_ok = test_bf16() if bf16_ok is None else bf16_ok
-        dtype = torch.bfloat16 if bf16_ok else torch.float16
-        dtype_vae = torch.bfloat16 if bf16_ok else torch.float16
-        dtype_unet = torch.bfloat16 if bf16_ok else torch.float16
-    elif shared.opts.cuda_dtype == 'FP16' or dtype == torch.float16:
-        fp16_ok = test_fp16() if fp16_ok is None else fp16_ok
-        bf16_ok = None
-        dtype = torch.float16 if fp16_ok else torch.float32
-        dtype_vae = torch.float16 if fp16_ok else torch.float32
-        dtype_unet = torch.float16 if fp16_ok else torch.float32
-    if shared.opts.no_half:
-        log.info('Torch override dtype: no-half set')
+    elif opts.cuda_dtype == 'BF16':
+        if not bf16_ok:
+            log.warning(f'Torch device capability failed: device={device} dtype={torch.bfloat16}')
+        dtype = torch.bfloat16
+        dtype_vae = torch.bfloat16
+        dtype_unet = torch.bfloat16
+    elif opts.cuda_dtype == 'FP16':
+        if not fp16_ok:
+            log.warning(f'Torch device capability failed: device={device} dtype={torch.float16}')
+        dtype = torch.float16
+        dtype_vae = torch.float16
+        dtype_unet = torch.float16
+
+    if opts.no_half:
        dtype = torch.float32
        dtype_vae = torch.float32
        dtype_unet = torch.float32
-    if shared.opts.no_half_vae: # set dtype again as no-half-vae options take priority
-        log.info('Torch override VAE dtype: no-half set')
+        log.info(f'Torch override: no-half dtype={dtype}')
+    if opts.no_half_vae:
        dtype_vae = torch.float32
-    unet_needs_upcast = shared.opts.upcast_sampling
-    if shared.opts.inference_mode == 'inference-mode':
+        log.info(f'Torch override: no-half-vae dtype={dtype_vae}')
+    unet_needs_upcast = opts.upcast_sampling
+    if opts.inference_mode == 'inference-mode':
        inference_context = torch.inference_mode
-    elif shared.opts.inference_mode == 'none':
+    elif opts.inference_mode == 'none':
        inference_context = contextlib.nullcontext
    else:
        inference_context = torch.no_grad
-    log_device_name = get_raw_openvino_device() if shared.cmd_opts.use_openvino else torch.device(get_optimal_device_name()) # pylint: disable=used-before-assignment
-    log.debug(f'Desired Torch parameters: dtype={shared.opts.cuda_dtype} no-half={shared.opts.no_half} no-half-vae={shared.opts.no_half_vae} upscast={shared.opts.upcast_sampling}')
-    log.info(f'Setting Torch parameters: device={log_device_name} dtype={dtype} vae={dtype_vae} unet={dtype_unet} context={inference_context.__name__} fp16={fp16_ok} bf16={bf16_ok} optimization={shared.opts.cross_attention_optimization}')


-args = cmd_args.parser.parse_args()
-backend = 'not set'
-if args.use_openvino:
-    from modules.intel.openvino import get_openvino_device
-    from modules.intel.openvino import get_device as get_raw_openvino_device
-    backend = 'openvino'
-    if hasattr(torch, 'xpu') and torch.xpu.is_available():
-        torch.xpu.is_available = lambda *args, **kwargs: False
-    torch.cuda.is_available = lambda *args, **kwargs: False
-elif args.use_ipex or (hasattr(torch, 'xpu') and torch.xpu.is_available()):
-    backend = 'ipex'
-    from modules.intel.ipex import ipex_init
-    ok, e = ipex_init()
-    if not ok:
-        log.error(f'IPEX initialization failed: {e}')
-        backend = 'cpu'
-elif args.use_directml:
-    backend = 'directml'
-    from modules.dml import directml_init
-    ok, e = directml_init()
-    if not ok:
-        log.error(f'DirectML initialization failed: {e}')
-        backend = 'cpu'
-elif torch.cuda.is_available() and torch.version.cuda:
-    backend = 'cuda'
-elif torch.cuda.is_available() and torch.version.hip:
-    backend = 'rocm'
-elif sys.platform == 'darwin':
-    backend = 'mps'
-else:
-    backend = 'cpu'
-
-
-inference_context = torch.no_grad
-cuda_ok = torch.cuda.is_available()
-cpu = torch.device("cpu")
-device = device_interrogate = device_gfpgan = device_esrgan = device_codeformer = None
-dtype = torch.float16
-dtype_vae = torch.float16
-dtype_unet = torch.float16
-fp16_ok = None
-bf16_ok = None
-unet_needs_upcast = False
-onnx = None
-if args.profile:
-    log.info(f'Torch build config: {torch.__config__.show()}')
-# set_cuda_sync_mode('block') # none/auto/spin/yield/block
+def set_cuda_params():
+    override_ipex_math()
+    set_cudnn_params()
+    set_sdpa_params()
+    set_dtype()
+    if backend == 'openvino':
+        from modules.intel.openvino import get_device as get_raw_openvino_device
+        device_name = get_raw_openvino_device()
+    else:
+        device_name = torch.device(get_optimal_device_name())
+    log.info(f'Torch parameters: backend={backend} device={device_name} config={opts.cuda_dtype} dtype={dtype} vae={dtype_vae} unet={dtype_unet} context={inference_context.__name__} nohalf={opts.no_half} nohalfvae={opts.no_half_vae} upscast={opts.upcast_sampling} deterministic={opts.cudnn_deterministic} test-fp16={fp16_ok} test-bf16={bf16_ok} optimization="{opts.cross_attention_optimization}"')


 def cond_cast_unet(tensor):
@@ -386,7 +452,7 @@ def randn(seed, shape=None):
        return None
    if device.type == 'mps':
        return torch.randn(shape, device=cpu).to(device)
-    elif shared.opts.diffusers_generator_device == "CPU":
+    elif opts.diffusers_generator_device == "CPU":
        return torch.randn(shape, device=cpu)
    else:
        return torch.randn(shape, device=device)
@@ -398,9 +464,9 @@ def randn_without_seed(shape):
    return torch.randn(shape, device=device)

 def autocast(disable=False):
-    if disable or dtype == torch.float32 or shared.cmd_opts.precision == "Full":
+    if disable or dtype == torch.float32:
        return contextlib.nullcontext()
-    if shared.cmd_opts.use_directml:
+    if backend == 'directml':
        return torch.dml.amp.autocast(dtype)
    if cuda_ok:
        return torch.autocast("cuda")
@@ -411,7 +477,7 @@ def autocast(disable=False):
 def without_autocast(disable=False):
    if disable:
        return contextlib.nullcontext()
-    if shared.cmd_opts.use_directml:
+    if backend == 'directml':
        return torch.dml.amp.autocast(enabled=False) if torch.is_autocast_enabled() else contextlib.nullcontext() # pylint: disable=unexpected-keyword-arg
    if cuda_ok:
        return torch.autocast("cuda", enabled=False) if torch.is_autocast_enabled() else contextlib.nullcontext()
@@ -424,19 +490,33 @@ class NansException(Exception):


 def test_for_nans(x, where):
-    if shared.opts.disable_nan_check:
+    if opts.disable_nan_check:
        return
    if not torch.all(torch.isnan(x)).item():
        return
    if where == "unet":
        message = "A tensor with all NaNs was produced in Unet."
-        if not shared.opts.no_half:
+        if not opts.no_half:
            message += " This could be either because there's not enough precision to represent the picture, or because your video card does not support half type. Try setting the \"Upcast cross attention layer to float32\" option in Settings > Stable Diffusion or using the --no-half commandline argument to fix this."
    elif where == "vae":
        message = "A tensor with all NaNs was produced in VAE."
-        if not shared.opts.no_half and not shared.opts.no_half_vae:
+        if not opts.no_half and not opts.no_half_vae:
            message += " This could be because there's not enough precision to represent the picture. Try adding --no-half-vae commandline argument to fix this."
    else:
        message = "A tensor with all NaNs was produced."
    message += " Use --disable-nan-check commandline argument to disable this check."
    raise NansException(message)
+
+
+def normalize_device(dev):
+    if torch.device(dev).type in {"cpu", "mps", "meta"}:
+        return torch.device(dev)
+    if torch.device(dev).index is None:
+        return torch.device(str(dev), index=0)
+    return torch.device(dev)
+
+
+def same_device(d1, d2):
+    if d1.type != d2.type:
+        return False
+    return normalize_device(d1) == normalize_device(d2)
@@ -72,8 +72,8 @@ def directml_do_hijack():
    from modules.devices import device

    CondFunc('torch.Generator',
-        lambda orig_func, device: orig_func("cpu"),
-        lambda orig_func, device: True)
+        lambda orig_func, device = None: orig_func("cpu"),
+        lambda orig_func, device = None: True)

    if not torch.dml.has_float64_support(device):
        torch.Tensor.__str__ = do_nothing_with_self
@@ -1,5 +1,5 @@
-import torch
 from typing import Optional
+import torch
 import transformers.models.clip.modeling_clip

 # Copied from transformers.models.bart.modeling_bart._make_causal_mask
@@ -17,7 +17,7 @@ console = Console(log_time=True, tab_size=4, log_time_format='%H:%M:%S-%f', soft
 }))

 pretty_install(console=console)
-traceback_install(console=console, extra_lines=1, width=console.width, word_wrap=False, indent_guides=False)
+traceback_install(console=console, extra_lines=1, width=console.width, word_wrap=False, indent_guides=False, max_frames=16)
 already_displayed = {}


@@ -38,7 +38,7 @@ def print_error_explanation(message):

 def display(e: Exception, task, suppress=[]):
    log.error(f"{task or 'error'}: {type(e).__name__}")
-    console.print_exception(show_locals=False, max_frames=10, extra_lines=1, suppress=suppress, theme="ansi_dark", word_wrap=False, width=console.width)
+    console.print_exception(show_locals=False, max_frames=16, extra_lines=1, suppress=suppress, theme="ansi_dark", word_wrap=False, width=console.width)


 def display_once(e: Exception, task):
@@ -56,7 +56,7 @@ def run(code, task):


 def exception(suppress=[]):
-    console.print_exception(show_locals=False, max_frames=10, extra_lines=2, suppress=suppress, theme="ansi_dark", word_wrap=False, width=min([console.width, 200]))
+    console.print_exception(show_locals=False, max_frames=16, extra_lines=2, suppress=suppress, theme="ansi_dark", word_wrap=False, width=min([console.width, 200]))


 def profile(profiler, msg: str):
@@ -286,7 +286,7 @@ def run_modelconvert(model, checkpoint_formats, precision, conv_type, custom_nam
    }
    shared.state.begin('Convert')
    model_info = sd_models.checkpoints_list[model]
-    shared.state.textinfo = f"Loading {model_info.filename}..."
+    shared.state.textinfo = f"Load {model_info.filename}..."
    shared.log.info(f"Model convert loading: {model_info.filename}")
    state_dict = load_model(model_info.filename)

@@ -134,7 +134,7 @@ class Script(scripts.Script):
            from modules.face.insightface import get_app
            app=get_app('buffalo_l')
            from modules.face.faceswap import face_swap
-            if shared.opts.save_images_before_face_restoration and not p.do_not_save_samples:
+            if shared.opts.save_images_before_detailer and not p.do_not_save_samples:
                for i, image in enumerate(processed.images):
                    info = processing.create_infotext(p, index=i)
                    images.save_image(image, path=p.outpath_samples, seed=p.all_seeds[i], prompt=p.all_prompts[i], info=info, p=p, suffix="-before-faceswap")
@@ -80,13 +80,13 @@ def face_id(
            basename, _ext = os.path.splitext(filename)
            model_path = hf.hf_hub_download(repo_id=folder, filename=filename, cache_dir=shared.opts.diffusers_dir)
            if model_path is None:
-                shared.log.error(f"FaceID download failed: model={model} file={ip_ckpt}")
+                shared.log.error(f'FaceID download failed: model={model} file="{ip_ckpt}"')
                return None
            if faceid_model_weights is None or faceid_model_name != model or not cache:
-                shared.log.debug(f"FaceID load: model={model} file={ip_ckpt}")
+                shared.log.debug(f'FaceID load: model={model} file="{ip_ckpt}"')
                faceid_model_weights = torch.load(model_path, map_location="cpu")
            else:
-                shared.log.debug(f"FaceID cached: model={model} file={ip_ckpt}")
+                shared.log.debug(f'FaceID cached: model={model} file="{ip_ckpt}"')

            if "XL Plus" in model and shared.sd_model_type == 'sd':
                image_encoder_path = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K"
@@ -238,9 +238,10 @@ def connect_paste(button, local_paste_fields, input_comp, override_settings_comp
                v = params.get(param_name, None)
                if v is None:
                    continue
-                if shared.opts.disable_weights_auto_swap:
-                    if setting_name == "sd_model_checkpoint" or setting_name == 'sd_model_refiner' or setting_name == 'sd_backend' or setting_name == 'sd_vae':
-                        continue
+                if setting_name == 'sd_backend':
+                    continue
+                if shared.opts.disable_weights_auto_swap and setting_name in ['sd_model_checkpoint', 'sd_model_refiner', 'sd_model_dict', 'sd_vae', 'sd_unet', 'sd_text_encoder']:
+                    continue
                v = shared.opts.cast_value(setting_name, v)
                current_value = getattr(shared.opts, setting_name, None)
                if v == current_value:
@@ -0,0 +1,21 @@
+from pathlib import Path
+import torch
+import gguf
+from .gguf_utils import TORCH_COMPATIBLE_QTYPES
+from .gguf_tensor import GGMLTensor
+
+
+def load_gguf_state_dict(path: str, compute_dtype: torch.dtype) -> dict[str, GGMLTensor]:
+    sd: dict[str, GGMLTensor] = {}
+    stats = {}
+    reader = gguf.GGUFReader(path)
+    for tensor in reader.tensors:
+        torch_tensor = torch.from_numpy(tensor.data)
+        shape = torch.Size(tuple(int(v) for v in reversed(tensor.shape)))
+        if tensor.tensor_type in TORCH_COMPATIBLE_QTYPES:
+            torch_tensor = torch_tensor.view(*shape)
+        sd[tensor.name] = GGMLTensor(torch_tensor, ggml_quantization_type=tensor.tensor_type, tensor_shape=shape, compute_dtype=compute_dtype)
+        if tensor.tensor_type.name not in stats:
+            stats[tensor.tensor_type.name] = 0
+        stats[tensor.tensor_type.name] += 1
+    return sd, stats
@@ -0,0 +1,151 @@
+# Original: invokeai.backend.quantization.gguf.ggml_tensor
+
+from typing import overload
+import torch
+import gguf
+from .gguf_utils import DEQUANTIZE_FUNCTIONS, TORCH_COMPATIBLE_QTYPES, dequantize
+
+
+def dequantize_and_run(func, args, kwargs):
+    """A helper function for running math ops on GGMLTensor inputs.
+
+    Dequantizes the inputs, and runs the function.
+    """
+    dequantized_args = [a.get_dequantized_tensor() if hasattr(a, "get_dequantized_tensor") else a for a in args]
+    dequantized_kwargs = {
+        k: v.get_dequantized_tensor() if hasattr(v, "get_dequantized_tensor") else v for k, v in kwargs.items()
+    }
+    return func(*dequantized_args, **dequantized_kwargs)
+
+
+def apply_to_quantized_tensor(func, args, kwargs):
+    """A helper function to apply a function to a quantized GGML tensor, and re-wrap the result in a GGMLTensor.
+
+    Assumes that the first argument is a GGMLTensor.
+    """
+    # We expect the first argument to be a GGMLTensor, and all other arguments to be non-GGMLTensors.
+    ggml_tensor = args[0]
+    assert isinstance(ggml_tensor, GGMLTensor)
+    assert all(not isinstance(a, GGMLTensor) for a in args[1:])
+    assert all(not isinstance(v, GGMLTensor) for v in kwargs.values())
+
+    new_data = func(ggml_tensor.quantized_data, *args[1:], **kwargs)
+
+    if new_data.dtype != ggml_tensor.quantized_data.dtype:
+        # This is intended to catch calls such as `.to(dtype-torch.float32)`, which are not supported on GGMLTensors.
+        raise ValueError("Operation changed the dtype of GGMLTensor unexpectedly.")
+
+    return GGMLTensor(
+        new_data, ggml_tensor._ggml_quantization_type, ggml_tensor.tensor_shape, ggml_tensor.compute_dtype
+    )
+
+
+GGML_TENSOR_OP_TABLE = {
+    # Ops to run on the quantized tensor.
+    torch.ops.aten.detach.default: apply_to_quantized_tensor,  # pyright: ignore
+    torch.ops.aten._to_copy.default: apply_to_quantized_tensor,  # pyright: ignore
+    # Ops to run on dequantized tensors.
+    torch.ops.aten.t.default: dequantize_and_run,  # pyright: ignore
+    torch.ops.aten.addmm.default: dequantize_and_run,  # pyright: ignore
+    torch.ops.aten.mul.Tensor: dequantize_and_run,  # pyright: ignore
+    torch.ops.aten.split.Tensor: dequantize_and_run,  # pyright: ignore
+}
+
+
+class GGMLTensor(torch.Tensor):
+    """A torch.Tensor sub-class holding a quantized GGML tensor.
+
+    The underlying tensor is quantized, but the GGMLTensor class provides a dequantized view of the tensor on-the-fly
+    when it is used in operations.
+    """
+
+    @staticmethod
+    def __new__(
+        cls,
+        data: torch.Tensor,
+        ggml_quantization_type: gguf.GGMLQuantizationType,
+        tensor_shape: torch.Size,
+        compute_dtype: torch.dtype,
+    ):
+        # Type hinting is not supported for torch.Tensor._make_wrapper_subclass, so we ignore the errors.
+        return torch.Tensor._make_wrapper_subclass(  # pyright: ignore
+            cls,
+            data.shape,
+            dtype=data.dtype,
+            layout=data.layout,
+            device=data.device,
+            strides=data.stride(),
+            storage_offset=data.storage_offset(),
+        )
+
+    def __init__(
+        self,
+        data: torch.Tensor,
+        ggml_quantization_type: gguf.GGMLQuantizationType,
+        tensor_shape: torch.Size,
+        compute_dtype: torch.dtype,
+    ):
+        self.quantized_data = data
+        self._ggml_quantization_type = ggml_quantization_type
+        # The dequantized shape of the tensor.
+        self.tensor_shape = tensor_shape
+        self.compute_dtype = compute_dtype
+
+    def __repr__(self, *, tensor_contents=None):
+        return f"GGMLTensor(type={self._ggml_quantization_type.name}, dequantized_shape=({self.tensor_shape})"
+
+    @overload
+    def size(self, dim: None = None) -> torch.Size: ...
+
+    @overload
+    def size(self, dim: int) -> int: ...
+
+    def size(self, dim: int | None = None):
+        """Return the size of the tensor after dequantization. I.e. the shape that will be used in any math ops."""
+        if dim is not None:
+            return self.tensor_shape[dim]
+        return self.tensor_shape
+
+    @property
+    def shape(self) -> torch.Size:  # pyright: ignore[reportIncompatibleVariableOverride] pyright doesn't understand this for some reason.
+        """The shape of the tensor after dequantization. I.e. the shape that will be used in any math ops."""
+        return self.size()
+
+    @property
+    def quantized_shape(self) -> torch.Size:
+        """The shape of the quantized tensor."""
+        return self.quantized_data.shape
+
+    def requires_grad_(self, mode: bool = True) -> torch.Tensor:
+        """The GGMLTensor class is currently only designed for inference (not training). Setting requires_grad to True
+        is not supported. This method is a no-op.
+        """
+        return self
+
+    def get_dequantized_tensor(self):
+        """Return the dequantized tensor.
+
+        Args:
+            dtype: The dtype of the dequantized tensor.
+        """
+        if self._ggml_quantization_type in TORCH_COMPATIBLE_QTYPES:
+            return self.quantized_data.to(self.compute_dtype)
+        elif self._ggml_quantization_type in DEQUANTIZE_FUNCTIONS:
+            # TODO(ryand): Look into how the dtype param is intended to be used.
+            return dequantize(
+                data=self.quantized_data, qtype=self._ggml_quantization_type, oshape=self.tensor_shape, dtype=None
+            ).to(self.compute_dtype)
+        else:
+            # There is no GPU implementation for this quantization type, so fallback to the numpy implementation.
+            new = gguf.quants.dequantize(self.quantized_data.cpu().numpy(), self._ggml_quantization_type)
+            return torch.from_numpy(new).to(self.quantized_data.device, dtype=self.compute_dtype)
+
+    @classmethod
+    def __torch_dispatch__(cls, func, types, args, kwargs):
+        # We will likely hit cases here in the future where a new op is encountered that is not yet supported.
+        # The new op simply needs to be added to the GGML_TENSOR_OP_TABLE.
+        if func in GGML_TENSOR_OP_TABLE:
+            return GGML_TENSOR_OP_TABLE[func](func, args, kwargs)
+        else:
+            return dequantize_and_run(func, args, kwargs)
+        return NotImplemented
@@ -0,0 +1,309 @@
+# Original: invokeai.backend.quantization.gguf.utils
+# Largely based on https://github.com/city96/ComfyUI-GGUF
+
+from typing import Callable, Optional, Union
+
+import gguf
+import torch
+
+TORCH_COMPATIBLE_QTYPES = {None, gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16}
+
+# K Quants #
+QK_K = 256
+K_SCALE_SIZE = 12
+
+
+def get_scale_min(scales: torch.Tensor):
+    n_blocks = scales.shape[0]
+    scales = scales.view(torch.uint8)
+    scales = scales.reshape((n_blocks, 3, 4))
+
+    d, m, m_d = torch.split(scales, scales.shape[-2] // 3, dim=-2)
+
+    sc = torch.cat([d & 0x3F, (m_d & 0x0F) | ((d >> 2) & 0x30)], dim=-1)
+    min = torch.cat([m & 0x3F, (m_d >> 4) | ((m >> 2) & 0x30)], dim=-1)
+
+    return (sc.reshape((n_blocks, 8)), min.reshape((n_blocks, 8)))
+
+
+# Legacy Quants #
+def dequantize_blocks_Q8_0(
+    blocks: torch.Tensor, block_size: int, type_size: int, dtype: Optional[torch.dtype] = None
+) -> torch.Tensor:
+    d, x = split_block_dims(blocks, 2)
+    d = d.view(torch.float16).to(dtype)
+    x = x.view(torch.int8)
+    return d * x
+
+
+def dequantize_blocks_Q5_1(
+    blocks: torch.Tensor, block_size: int, type_size: int, dtype: Optional[torch.dtype] = None
+) -> torch.Tensor:
+    n_blocks = blocks.shape[0]
+
+    d, m, qh, qs = split_block_dims(blocks, 2, 2, 4)
+    d = d.view(torch.float16).to(dtype)
+    m = m.view(torch.float16).to(dtype)
+    qh = to_uint32(qh)
+
+    qh = qh.reshape((n_blocks, 1)) >> torch.arange(32, device=d.device, dtype=torch.int32).reshape(1, 32)
+    ql = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor(
+        [0, 4], device=d.device, dtype=torch.uint8
+    ).reshape(1, 1, 2, 1)
+    qh = (qh & 1).to(torch.uint8)
+    ql = (ql & 0x0F).reshape((n_blocks, -1))
+
+    qs = ql | (qh << 4)
+    return (d * qs) + m
+
+
+def dequantize_blocks_Q5_0(
+    blocks: torch.Tensor, block_size: int, type_size: int, dtype: Optional[torch.dtype] = None
+) -> torch.Tensor:
+    n_blocks = blocks.shape[0]
+
+    d, qh, qs = split_block_dims(blocks, 2, 4)
+    d = d.view(torch.float16).to(dtype)
+    qh = to_uint32(qh)
+
+    qh = qh.reshape(n_blocks, 1) >> torch.arange(32, device=d.device, dtype=torch.int32).reshape(1, 32)
+    ql = qs.reshape(n_blocks, -1, 1, block_size // 2) >> torch.tensor(
+        [0, 4], device=d.device, dtype=torch.uint8
+    ).reshape(1, 1, 2, 1)
+
+    qh = (qh & 1).to(torch.uint8)
+    ql = (ql & 0x0F).reshape(n_blocks, -1)
+
+    qs = (ql | (qh << 4)).to(torch.int8) - 16
+    return d * qs
+
+
+def dequantize_blocks_Q4_1(
+    blocks: torch.Tensor, block_size: int, type_size: int, dtype: Optional[torch.dtype] = None
+) -> torch.Tensor:
+    n_blocks = blocks.shape[0]
+
+    d, m, qs = split_block_dims(blocks, 2, 2)
+    d = d.view(torch.float16).to(dtype)
+    m = m.view(torch.float16).to(dtype)
+
+    qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor(
+        [0, 4], device=d.device, dtype=torch.uint8
+    ).reshape(1, 1, 2, 1)
+    qs = (qs & 0x0F).reshape(n_blocks, -1)
+
+    return (d * qs) + m
+
+
+def dequantize_blocks_Q4_0(
+    blocks: torch.Tensor, block_size: int, type_size: int, dtype: Optional[torch.dtype] = None
+) -> torch.Tensor:
+    n_blocks = blocks.shape[0]
+
+    d, qs = split_block_dims(blocks, 2)
+    d = d.view(torch.float16).to(dtype)
+
+    qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor(
+        [0, 4], device=d.device, dtype=torch.uint8
+    ).reshape((1, 1, 2, 1))
+    qs = (qs & 0x0F).reshape((n_blocks, -1)).to(torch.int8) - 8
+    return d * qs
+
+
+def dequantize_blocks_BF16(
+    blocks: torch.Tensor, block_size: int, type_size: int, dtype: Optional[torch.dtype] = None
+) -> torch.Tensor:
+    return (blocks.view(torch.int16).to(torch.int32) << 16).view(torch.float32)
+
+
+def dequantize_blocks_Q6_K(
+    blocks: torch.Tensor, block_size: int, type_size: int, dtype: Optional[torch.dtype] = None
+) -> torch.Tensor:
+    n_blocks = blocks.shape[0]
+
+    (
+        ql,
+        qh,
+        scales,
+        d,
+    ) = split_block_dims(blocks, QK_K // 2, QK_K // 4, QK_K // 16)
+
+    scales = scales.view(torch.int8).to(dtype)
+    d = d.view(torch.float16).to(dtype)
+    d = (d * scales).reshape((n_blocks, QK_K // 16, 1))
+
+    ql = ql.reshape((n_blocks, -1, 1, 64)) >> torch.tensor([0, 4], device=d.device, dtype=torch.uint8).reshape(
+        (1, 1, 2, 1)
+    )
+    ql = (ql & 0x0F).reshape((n_blocks, -1, 32))
+    qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8).reshape(
+        (1, 1, 4, 1)
+    )
+    qh = (qh & 0x03).reshape((n_blocks, -1, 32))
+    q = (ql | (qh << 4)).to(torch.int8) - 32
+    q = q.reshape((n_blocks, QK_K // 16, -1))
+
+    return (d * q).reshape((n_blocks, QK_K))
+
+
+def dequantize_blocks_Q5_K(
+    blocks: torch.Tensor, block_size: int, type_size: int, dtype: Optional[torch.dtype] = None
+) -> torch.Tensor:
+    n_blocks = blocks.shape[0]
+
+    d, dmin, scales, qh, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE, QK_K // 8)
+
+    d = d.view(torch.float16).to(dtype)
+    dmin = dmin.view(torch.float16).to(dtype)
+
+    sc, m = get_scale_min(scales)
+
+    d = (d * sc).reshape((n_blocks, -1, 1))
+    dm = (dmin * m).reshape((n_blocks, -1, 1))
+
+    ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d.device, dtype=torch.uint8).reshape(
+        (1, 1, 2, 1)
+    )
+    qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor(list(range(8)), device=d.device, dtype=torch.uint8).reshape(
+        (1, 1, 8, 1)
+    )
+    ql = (ql & 0x0F).reshape((n_blocks, -1, 32))
+    qh = (qh & 0x01).reshape((n_blocks, -1, 32))
+    q = ql | (qh << 4)
+
+    return (d * q - dm).reshape((n_blocks, QK_K))
+
+
+def dequantize_blocks_Q4_K(
+    blocks: torch.Tensor, block_size: int, type_size: int, dtype: Optional[torch.dtype] = None
+) -> torch.Tensor:
+    n_blocks = blocks.shape[0]
+
+    d, dmin, scales, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE)
+    d = d.view(torch.float16).to(dtype)
+    dmin = dmin.view(torch.float16).to(dtype)
+
+    sc, m = get_scale_min(scales)
+
+    d = (d * sc).reshape((n_blocks, -1, 1))
+    dm = (dmin * m).reshape((n_blocks, -1, 1))
+
+    qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d.device, dtype=torch.uint8).reshape(
+        (1, 1, 2, 1)
+    )
+    qs = (qs & 0x0F).reshape((n_blocks, -1, 32))
+
+    return (d * qs - dm).reshape((n_blocks, QK_K))
+
+
+def dequantize_blocks_Q3_K(
+    blocks: torch.Tensor, block_size: int, type_size: int, dtype: Optional[torch.dtype] = None
+) -> torch.Tensor:
+    n_blocks = blocks.shape[0]
+
+    hmask, qs, scales, d = split_block_dims(blocks, QK_K // 8, QK_K // 4, 12)
+    d = d.view(torch.float16).to(dtype)
+
+    lscales, hscales = scales[:, :8], scales[:, 8:]
+    lscales = lscales.reshape((n_blocks, 1, 8)) >> torch.tensor([0, 4], device=d.device, dtype=torch.uint8).reshape(
+        (1, 2, 1)
+    )
+    lscales = lscales.reshape((n_blocks, 16))
+    hscales = hscales.reshape((n_blocks, 1, 4)) >> torch.tensor(
+        [0, 2, 4, 6], device=d.device, dtype=torch.uint8
+    ).reshape((1, 4, 1))
+    hscales = hscales.reshape((n_blocks, 16))
+    scales = (lscales & 0x0F) | ((hscales & 0x03) << 4)
+    scales = scales.to(torch.int8) - 32
+
+    dl = (d * scales).reshape((n_blocks, 16, 1))
+
+    ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8).reshape(
+        (1, 1, 4, 1)
+    )
+    qh = hmask.reshape(n_blocks, -1, 1, 32) >> torch.tensor(list(range(8)), device=d.device, dtype=torch.uint8).reshape(
+        (1, 1, 8, 1)
+    )
+    ql = ql.reshape((n_blocks, 16, QK_K // 16)) & 3
+    qh = (qh.reshape((n_blocks, 16, QK_K // 16)) & 1) ^ 1
+    q = ql.to(torch.int8) - (qh << 2).to(torch.int8)
+
+    return (dl * q).reshape((n_blocks, QK_K))
+
+
+def dequantize_blocks_Q2_K(
+    blocks: torch.Tensor, block_size: int, type_size: int, dtype: Optional[torch.dtype] = None
+) -> torch.Tensor:
+    n_blocks = blocks.shape[0]
+
+    scales, qs, d, dmin = split_block_dims(blocks, QK_K // 16, QK_K // 4, 2)
+    d = d.view(torch.float16).to(dtype)
+    dmin = dmin.view(torch.float16).to(dtype)
+
+    # (n_blocks, 16, 1)
+    dl = (d * (scales & 0xF)).reshape((n_blocks, QK_K // 16, 1))
+    ml = (dmin * (scales >> 4)).reshape((n_blocks, QK_K // 16, 1))
+
+    shift = torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
+
+    qs = (qs.reshape((n_blocks, -1, 1, 32)) >> shift) & 3
+    qs = qs.reshape((n_blocks, QK_K // 16, 16))
+    qs = dl * qs - ml
+
+    return qs.reshape((n_blocks, -1))
+
+
+DEQUANTIZE_FUNCTIONS: dict[
+    gguf.GGMLQuantizationType, Callable[[torch.Tensor, int, int, Optional[torch.dtype]], torch.Tensor]
+] = {
+    gguf.GGMLQuantizationType.BF16: dequantize_blocks_BF16,
+    gguf.GGMLQuantizationType.Q8_0: dequantize_blocks_Q8_0,
+    gguf.GGMLQuantizationType.Q5_1: dequantize_blocks_Q5_1,
+    gguf.GGMLQuantizationType.Q5_0: dequantize_blocks_Q5_0,
+    gguf.GGMLQuantizationType.Q4_1: dequantize_blocks_Q4_1,
+    gguf.GGMLQuantizationType.Q4_0: dequantize_blocks_Q4_0,
+    gguf.GGMLQuantizationType.Q6_K: dequantize_blocks_Q6_K,
+    gguf.GGMLQuantizationType.Q5_K: dequantize_blocks_Q5_K,
+    gguf.GGMLQuantizationType.Q4_K: dequantize_blocks_Q4_K,
+    gguf.GGMLQuantizationType.Q3_K: dequantize_blocks_Q3_K,
+    gguf.GGMLQuantizationType.Q2_K: dequantize_blocks_Q2_K,
+}
+
+
+def is_torch_compatible(tensor: Optional[torch.Tensor]):
+    return getattr(tensor, "tensor_type", None) in TORCH_COMPATIBLE_QTYPES
+
+
+def is_quantized(tensor: torch.Tensor):
+    return not is_torch_compatible(tensor)
+
+
+def dequantize(
+    data: torch.Tensor, qtype: gguf.GGMLQuantizationType, oshape: torch.Size, dtype: Optional[torch.dtype] = None
+):
+    """
+    Dequantize tensor back to usable shape/dtype
+    """
+    block_size, type_size = gguf.GGML_QUANT_SIZES[qtype]
+    dequantize_blocks = DEQUANTIZE_FUNCTIONS[qtype]
+
+    rows = data.reshape((-1, data.shape[-1])).view(torch.uint8)
+
+    n_blocks = rows.numel() // type_size
+    blocks = rows.reshape((n_blocks, type_size))
+    blocks = dequantize_blocks(blocks, block_size, type_size, dtype)
+    return blocks.reshape(oshape)
+
+
+def to_uint32(x: torch.Tensor) -> torch.Tensor:
+    x = x.view(torch.uint8).to(torch.int32)
+    return (x[:, 0] | x[:, 1] << 8 | x[:, 2] << 16 | x[:, 3] << 24).unsqueeze(1)
+
+
+def split_block_dims(blocks: torch.Tensor, *args):
+    n_max = blocks.shape[1]
+    dims = list(args) + [n_max - sum(args)]
+    return torch.split(blocks, dims, dim=1)
+
+
+PATCH_TYPES = Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor]]
@@ -0,0 +1,82 @@
+"""
+TODO:
+- apply metadata
+- preview
+- load/save
+"""
+
+import sys
+import datetime
+from collections import deque
+import torch
+from modules import shared, devices
+
+
+class Item():
+    def __init__(self, latent, preview=None, info=None, ops=[]):
+        self.ts = datetime.datetime.now().replace(microsecond=0)
+        self.name = self.ts.strftime('%Y-%m-%d %H:%M:%S')
+        self.latent = latent.detach().clone().to(devices.cpu)
+        self.preview = preview
+        self.info = info
+        self.ops = ops.copy()
+        self.size = sys.getsizeof(self.latent.storage())
+
+
+class History():
+    def __init__(self):
+        self.index = -1
+        self.latents = deque(maxlen=1024)
+
+    @property
+    def count(self):
+        return len(self.latents)
+
+    @property
+    def size(self):
+        s = 0
+        for item in self.latents:
+            s += item.size
+        return s
+
+    @property
+    def list(self):
+        shared.log.info(f'History: items={self.count}/{shared.opts.latent_history} size={self.size}')
+        return [item.name for item in self.latents]
+
+    @property
+    def selected(self):
+        if self.index >= 0 and self.index < self.count:
+            index = self.index
+            self.index = -1
+        else:
+            index = 0
+        item = self.latents[index]
+        shared.log.debug(f'History get: index={index} time={item.ts} shape={item.latent.shape} dtype={item.latent.dtype} count={self.count}')
+        return item.latent.to(devices.device), index
+
+    def find(self, name):
+        for i, item in enumerate(self.latents):
+            if item.name == name:
+                return i
+        return -1
+
+    def add(self, latent, preview=None, info=None, ops=[]):
+        if shared.opts.latent_history == 0:
+            return
+        if torch.is_tensor(latent):
+            item = Item(latent, preview, info, ops)
+            self.latents.appendleft(item)
+            # shared.log.debug(f'History add: shape={latent.shape} dtype={latent.dtype} count={self.count}')
+            if self.count >= shared.opts.latent_history:
+                self.latents.pop()
+
+    def clear(self):
+        self.latents.clear()
+        # shared.log.debug(f'History clear: count={self.count}')
+
+    def load(self):
+        pass
+
+    def save(self):
+        pass
@@ -2,23 +2,19 @@ import io
 import re
 import os
 import sys
-import math
 import json
-import uuid
-import time
 import queue
-import string
 import random
-import hashlib
 import datetime
 import threading
-from pathlib import Path
-from collections import namedtuple
 import numpy as np
 import piexif
 import piexif.helper
-from PIL import Image, ImageFont, ImageDraw, PngImagePlugin, ExifTags
+from PIL import Image, PngImagePlugin, ExifTags
 from modules import sd_samplers, shared, script_callbacks, errors, paths
+from modules.images_grid import image_grid, get_grid_size, split_grid, combine_grid, check_grid_size, get_font, draw_grid_annotations, draw_prompt_matrix, GridAnnotation, Grid # pylint: disable=unused-import
+from modules.images_resize import resize_image # pylint: disable=unused-import
+from modules.images_namegen import FilenameGenerator, get_next_sequence_number # pylint: disable=unused-import


 debug = errors.log.trace if os.environ.get('SD_PATH_DEBUG', None) is not None else lambda *args, **kwargs: None
@@ -29,550 +25,6 @@ except Exception:
    pass


-def check_grid_size(imgs):
-    mp = 0
-    for img in imgs:
-        mp += img.width * img.height if img is not None else 0
-    mp = round(mp / 1000000)
-    ok = mp <= shared.opts.img_max_size_mp
-    if not ok:
-        shared.log.warning(f'Maximum image size exceded: size={mp} maximum={shared.opts.img_max_size_mp} MPixels')
-    return ok
-
-
-def image_grid(imgs, batch_size=1, rows=None):
-    if rows is None:
-        if shared.opts.n_rows > 0:
-            rows = shared.opts.n_rows
-        elif shared.opts.n_rows == 0:
-            rows = batch_size
-        else:
-            rows = math.floor(math.sqrt(len(imgs)))
-            while len(imgs) % rows != 0:
-                rows -= 1
-    if rows > len(imgs):
-        rows = len(imgs)
-    cols = math.ceil(len(imgs) / rows)
-    params = script_callbacks.ImageGridLoopParams(imgs, cols, rows)
-    script_callbacks.image_grid_callback(params)
-    imgs = [i for i in imgs if i is not None] if imgs is not None else []
-    if len(imgs) == 0:
-        return None
-    w, h = imgs[0].size
-    grid = Image.new('RGB', size=(params.cols * w, params.rows * h), color=shared.opts.grid_background)
-    for i, img in enumerate(params.imgs):
-        grid.paste(img, box=(i % params.cols * w, i // params.cols * h))
-    return grid
-
-
-Grid = namedtuple("Grid", ["tiles", "tile_w", "tile_h", "image_w", "image_h", "overlap"])
-
-
-def split_grid(image, tile_w=512, tile_h=512, overlap=64):
-    w = image.width
-    h = image.height
-    non_overlap_width = tile_w - overlap
-    non_overlap_height = tile_h - overlap
-    cols = math.ceil((w - overlap) / non_overlap_width)
-    rows = math.ceil((h - overlap) / non_overlap_height)
-    dx = (w - tile_w) / (cols - 1) if cols > 1 else 0
-    dy = (h - tile_h) / (rows - 1) if rows > 1 else 0
-    grid = Grid([], tile_w, tile_h, w, h, overlap)
-    for row in range(rows):
-        row_images = []
-        y = int(row * dy)
-        if y + tile_h >= h:
-            y = h - tile_h
-        for col in range(cols):
-            x = int(col * dx)
-            if x + tile_w >= w:
-                x = w - tile_w
-            tile = image.crop((x, y, x + tile_w, y + tile_h))
-            row_images.append([x, tile_w, tile])
-        grid.tiles.append([y, tile_h, row_images])
-    return grid
-
-
-def combine_grid(grid):
-    def make_mask_image(r):
-        r = r * 255 / grid.overlap
-        r = r.astype(np.uint8)
-        return Image.fromarray(r, 'L')
-
-    mask_w = make_mask_image(np.arange(grid.overlap, dtype=np.float32).reshape((1, grid.overlap)).repeat(grid.tile_h, axis=0))
-    mask_h = make_mask_image(np.arange(grid.overlap, dtype=np.float32).reshape((grid.overlap, 1)).repeat(grid.image_w, axis=1))
-    combined_image = Image.new("RGB", (grid.image_w, grid.image_h))
-    for y, h, row in grid.tiles:
-        combined_row = Image.new("RGB", (grid.image_w, h))
-        for x, w, tile in row:
-            if x == 0:
-                combined_row.paste(tile, (0, 0))
-                continue
-            combined_row.paste(tile.crop((0, 0, grid.overlap, h)), (x, 0), mask=mask_w)
-            combined_row.paste(tile.crop((grid.overlap, 0, w, h)), (x + grid.overlap, 0))
-        if y == 0:
-            combined_image.paste(combined_row, (0, 0))
-            continue
-        combined_image.paste(combined_row.crop((0, 0, combined_row.width, grid.overlap)), (0, y), mask=mask_h)
-        combined_image.paste(combined_row.crop((0, grid.overlap, combined_row.width, h)), (0, y + grid.overlap))
-    return combined_image
-
-
-class GridAnnotation:
-    def __init__(self, text='', is_active=True):
-        self.text = text
-        self.is_active = is_active
-        self.size = None
-
-
-def get_font(fontsize):
-    try:
-        return ImageFont.truetype(shared.opts.font or "javascript/notosans-nerdfont-regular.ttf", fontsize)
-    except Exception:
-        return ImageFont.truetype("javascript/notosans-nerdfont-regular.ttf", fontsize)
-
-
-def draw_grid_annotations(im, width, height, hor_texts, ver_texts, margin=0, title=None):
-    def wrap(drawing, text, font, line_length):
-        lines = ['']
-        for word in text.split():
-            line = f'{lines[-1]} {word}'.strip()
-            if drawing.textlength(line, font=font) <= line_length:
-                lines[-1] = line
-            else:
-                lines.append(word)
-        return lines
-
-    def draw_texts(drawing: ImageDraw, draw_x, draw_y, lines, initial_fnt, initial_fontsize):
-        for line in lines:
-            font = initial_fnt
-            fontsize = initial_fontsize
-            while drawing.multiline_textbbox((0,0), text=line.text, font=font)[2] > line.allowed_width and fontsize > 0:
-                fontsize -= 1
-                font = get_font(fontsize)
-            drawing.multiline_text((draw_x, draw_y + line.size[1] / 2), line.text, font=font, fill=shared.opts.font_color if line.is_active else color_inactive, anchor="mm", align="center")
-            if not line.is_active:
-                drawing.line((draw_x - line.size[0] // 2, draw_y + line.size[1] // 2, draw_x + line.size[0] // 2, draw_y + line.size[1] // 2), fill=color_inactive, width=4)
-            draw_y += line.size[1] + line_spacing
-
-    fontsize = (width + height) // 25
-    line_spacing = fontsize // 2
-    font = get_font(fontsize)
-    color_inactive = (127, 127, 127)
-    pad_left = 0 if sum([sum([len(line.text) for line in lines]) for lines in ver_texts]) == 0 else width * 3 // 4
-    cols = im.width // width
-    rows = im.height // height
-    assert cols == len(hor_texts), f'bad number of horizontal texts: {len(hor_texts)}; must be {cols}'
-    assert rows == len(ver_texts), f'bad number of vertical texts: {len(ver_texts)}; must be {rows}'
-    calc_img = Image.new("RGB", (1, 1), shared.opts.grid_background)
-    calc_d = ImageDraw.Draw(calc_img)
-    title_texts = [title] if title else [[GridAnnotation()]]
-    for texts, allowed_width in zip(hor_texts + ver_texts + title_texts, [width] * len(hor_texts) + [pad_left] * len(ver_texts) + [(width+margin)*cols]):
-        items = [] + texts
-        texts.clear()
-        for line in items:
-            wrapped = wrap(calc_d, line.text, font, allowed_width)
-            texts += [GridAnnotation(x, line.is_active) for x in wrapped]
-        for line in texts:
-            bbox = calc_d.multiline_textbbox((0, 0), line.text, font=font)
-            line.size = (bbox[2] - bbox[0], bbox[3] - bbox[1])
-            line.allowed_width = allowed_width
-    hor_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing for lines in hor_texts]
-    ver_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing * len(lines) for lines in ver_texts]
-    pad_top = 0 if sum(hor_text_heights) == 0 else max(hor_text_heights) + line_spacing * 2
-    title_pad = 0
-    if title:
-        title_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing for lines in title_texts] # pylint: disable=unsubscriptable-object
-        title_pad = 0 if sum(title_text_heights) == 0 else max(title_text_heights) + line_spacing * 2
-    result = Image.new("RGB", (im.width + pad_left + margin * (cols-1), im.height + pad_top + title_pad + margin * (rows-1)), shared.opts.grid_background)
-    for row in range(rows):
-        for col in range(cols):
-            cell = im.crop((width * col, height * row, width * (col+1), height * (row+1)))
-            result.paste(cell, (pad_left + (width + margin) * col, pad_top + title_pad + (height + margin) * row))
-    d = ImageDraw.Draw(result)
-    if title:
-        x = pad_left + ((width+margin)*cols) / 2
-        y = title_pad / 2 - title_text_heights[0] / 2
-        draw_texts(d, x, y, title_texts[0], font, fontsize)
-    for col in range(cols):
-        x = pad_left + (width + margin) * col + width / 2
-        y = (pad_top / 2 - hor_text_heights[col] / 2) + title_pad
-        draw_texts(d, x, y, hor_texts[col], font, fontsize)
-    for row in range(rows):
-        x = pad_left / 2
-        y = (pad_top + (height + margin) * row + height / 2 - ver_text_heights[row] / 2) + title_pad
-        draw_texts(d, x, y, ver_texts[row], font, fontsize)
-    return result
-
-
-def draw_prompt_matrix(im, width, height, all_prompts, margin=0):
-    prompts = all_prompts[1:]
-    boundary = math.ceil(len(prompts) / 2)
-    prompts_horiz = prompts[:boundary]
-    prompts_vert = prompts[boundary:]
-    hor_texts = [[GridAnnotation(x, is_active=pos & (1 << i) != 0) for i, x in enumerate(prompts_horiz)] for pos in range(1 << len(prompts_horiz))]
-    ver_texts = [[GridAnnotation(x, is_active=pos & (1 << i) != 0) for i, x in enumerate(prompts_vert)] for pos in range(1 << len(prompts_vert))]
-    return draw_grid_annotations(im, width, height, hor_texts, ver_texts, margin)
-
-
-def resize_image(resize_mode, im, width, height, upscaler_name=None, output_type='image', context=None):
-    upscaler_name = upscaler_name or shared.opts.upscaler_for_img2img
-
-    def latent(im, w, h, upscaler):
-        from modules.processing_vae import vae_encode, vae_decode
-        import torch
-        latents = vae_encode(im, shared.sd_model, full_quality=False) # TODO enable full VAE mode for resize-latent
-        latents = torch.nn.functional.interpolate(latents, size=(int(h // 8), int(w // 8)), mode=upscaler["mode"], antialias=upscaler["antialias"])
-        im = vae_decode(latents, shared.sd_model, output_type='pil', full_quality=False)[0]
-        return im
-
-    def resize(im, w, h):
-        w = int(w)
-        h = int(h)
-        if upscaler_name is None or upscaler_name == "None" or im.mode == 'L':
-            return im.resize((w, h), resample=Image.Resampling.LANCZOS) # force for mask
-        scale = max(w / im.width, h / im.height)
-        if scale > 1.0:
-            upscalers = [x for x in shared.sd_upscalers if x.name.lower().replace('-', ' ') == upscaler_name.lower().replace('-', ' ')]
-            if len(upscalers) > 0:
-                upscaler = upscalers[0]
-                im = upscaler.scaler.upscale(im, scale, upscaler.data_path)
-            else:
-                upscaler = shared.latent_upscale_modes.get(upscaler_name, None)
-                if upscaler is not None:
-                    im = latent(im, w, h, upscaler)
-                else:
-                    upscaler = shared.sd_upscalers[0]
-                    shared.log.warning(f"Resize upscaler: invalid={upscaler_name} fallback={upscaler.name}")
-                    shared.log.debug(f"Resize upscaler: available={[u.name for u in shared.sd_upscalers]}")
-        if im.width != w or im.height != h: # probably downsample after upscaler created larger image
-            im = im.resize((w, h), resample=Image.Resampling.LANCZOS)
-        return im
-
-    def crop(im):
-        ratio = width / height
-        src_ratio = im.width / im.height
-        src_w = width if ratio > src_ratio else im.width * height // im.height
-        src_h = height if ratio <= src_ratio else im.height * width // im.width
-        resized = resize(im, src_w, src_h)
-        res = Image.new(im.mode, (width, height))
-        res.paste(resized, box=(width // 2 - src_w // 2, height // 2 - src_h // 2))
-        return res
-
-    def fill(im, color=None):
-        color = color or shared.opts.image_background
-        """
-        ratio = round(width / height, 1)
-        src_ratio = round(im.width / im.height, 1)
-        src_w = width if ratio < src_ratio else im.width * height // im.height
-        src_h = height if ratio >= src_ratio else im.height * width // im.width
-        resized = resize(im, src_w, src_h)
-        res = Image.new(im.mode, (width, height))
-        res.paste(resized, box=(width // 2 - src_w // 2, height // 2 - src_h // 2))
-        if ratio < src_ratio:
-            fill_height = height // 2 - src_h // 2
-            if width > 0 and fill_height > 0:
-                res.paste(resized.resize((width, fill_height), box=(0, 0, width, 0)), box=(0, 0))
-                res.paste(resized.resize((width, fill_height), box=(0, resized.height, width, resized.height)), box=(0, fill_height + src_h))
-        elif ratio > src_ratio:
-            fill_width = width // 2 - src_w // 2
-            if height > 0 and fill_width > 0:
-                res.paste(resized.resize((fill_width, height), box=(0, 0, 0, height)), box=(0, 0))
-                res.paste(resized.resize((fill_width, height), box=(resized.width, 0, resized.width, height)), box=(fill_width + src_w, 0))
-        return res
-        """
-        ratio = min(width / im.width, height / im.height)
-        im = resize(im, int(im.width * ratio), int(im.height * ratio))
-        res = Image.new(im.mode, (width, height), color=color)
-        res.paste(im, box=((width - im.width)//2, (height - im.height)//2))
-        return res
-
-    def context_aware(im, width, height, context):
-        import seam_carving # https://github.com/li-plus/seam-carving
-        if 'forward' in context:
-            energy_mode = "forward"
-        elif 'backward' in context:
-            energy_mode = "backward"
-        else:
-            return im
-        if 'Add' in context:
-            src_ratio = min(width / im.width, height / im.height)
-            src_w = int(im.width * src_ratio)
-            src_h = int(im.height * src_ratio)
-            src_image = resize(im, src_w, src_h)
-        elif 'Remove' in context:
-            ratio = width / height
-            src_ratio = im.width / im.height
-            src_w = width if ratio > src_ratio else im.width * height // im.height
-            src_h = height if ratio <= src_ratio else im.height * width // im.width
-            src_image = resize(im, src_w, src_h)
-        else:
-            return im
-        res = Image.fromarray(seam_carving.resize(
-            src_image, # source image (rgb or gray)
-            size=(width, height),  # target size
-            energy_mode=energy_mode,  # choose from {backward, forward}
-            order="width-first",  # choose from {width-first, height-first}
-            keep_mask=None,  # object mask to protect from removal
-        ))
-        return res
-
-    t0 = time.time()
-    if resize_mode is None:
-        resize_mode = 0
-    if resize_mode == 0 or (im.width == width and im.height == height) or (width == 0 and height == 0): # none
-        res = im.copy()
-    elif resize_mode == 1: # fixed
-        res = resize(im, width, height)
-    elif resize_mode == 2: # crop
-        res = crop(im)
-    elif resize_mode == 3: # fill
-        res = fill(im)
-    elif resize_mode == 4: # edge
-        from modules import masking
-        res = fill(im, color=0)
-        res, _mask = masking.outpaint(res)
-    elif resize_mode == 5:  # context-aware
-        res = context_aware(im, width, height, context)
-    else:
-        res = im.copy()
-        shared.log.error(f'Invalid resize mode: {resize_mode}')
-    t1 = time.time()
-    shared.log.debug(f'Image resize: input={im} width={width} height={height} mode="{shared.resize_modes[resize_mode]}" upscaler="{upscaler_name}" context="{context}" type={output_type} result={res} time={t1-t0:.2f} fn={sys._getframe(1).f_code.co_filename}:{sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access
-    return np.array(res) if output_type == 'np' else res
-
-
-re_nonletters = re.compile(r'[\s' + string.punctuation + ']+')
-re_pattern = re.compile(r"(.*?)(?:\[([^\[\]]+)\]|$)")
-re_pattern_arg = re.compile(r"(.*)<([^>]*)>$")
-re_attention = re.compile(r'[\(*\[*](\w+)(:\d+(\.\d+))?[\)*\]*]|')
-re_network = re.compile(r'\<\w+:(\w+)(:\d+(\.\d+))?\>|')
-re_brackets = re.compile(r'[\([{})\]]')
-
-NOTHING = object()
-
-
-class FilenameGenerator:
-    replacements = {
-        'width': lambda self: self.image.width,
-        'height': lambda self: self.image.height,
-        'batch_number': lambda self: self.batch_number,
-        'iter_number': lambda self: self.iter_number,
-        'num': lambda self: NOTHING if self.p.n_iter == 1 and self.p.batch_size == 1 else self.p.iteration * self.p.batch_size + self.p.batch_index + 1,
-        'generation_number': lambda self: NOTHING if self.p.n_iter == 1 and self.p.batch_size == 1 else self.p.iteration * self.p.batch_size + self.p.batch_index + 1,
-        'date': lambda self: datetime.datetime.now().strftime('%Y-%m-%d'),
-        'datetime': lambda self, *args: self.datetime(*args),  # accepts formats: [datetime], [datetime<Format>], [datetime<Format><Time Zone>]
-        'hasprompt': lambda self, *args: self.hasprompt(*args),  # accepts formats:[hasprompt<prompt1|default><prompt2>..]
-        'hash': lambda self: self.image_hash(),
-        'image_hash': lambda self: self.image_hash(),
-        'timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp),
-        'job_timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp),
-
-        'model': lambda self: shared.sd_model.sd_checkpoint_info.title,
-        'model_shortname': lambda self: shared.sd_model.sd_checkpoint_info.model_name,
-        'model_name': lambda self: shared.sd_model.sd_checkpoint_info.model_name,
-        'model_hash': lambda self: shared.sd_model.sd_checkpoint_info.shorthash,
-
-        'prompt': lambda self: self.prompt_full(),
-        'prompt_no_styles': lambda self: self.prompt_no_style(),
-        'prompt_words': lambda self: self.prompt_words(),
-        'prompt_hash': lambda self: hashlib.sha256(self.prompt.encode()).hexdigest()[0:8],
-
-        'sampler': lambda self: self.p and self.p.sampler_name,
-        'seed': lambda self: self.seed and str(self.seed) or '',
-        'steps': lambda self: self.p and getattr(self.p, 'steps', 0),
-        'cfg': lambda self: self.p and getattr(self.p, 'cfg_scale', 0),
-        'clip_skip': lambda self: self.p and getattr(self.p, 'clip_skip', 0),
-        'denoising': lambda self: self.p and getattr(self.p, 'denoising_strength', 0),
-        'styles': lambda self: self.p and ", ".join([style for style in self.p.styles if not style == "None"]) or "None",
-        'uuid': lambda self: str(uuid.uuid4()),
-    }
-    default_time_format = '%Y%m%d%H%M%S'
-
-    def __init__(self, p, seed, prompt, image, grid=False):
-        if p is None:
-            debug('Filename generator init skip')
-        else:
-            debug(f'Filename generator init: {seed} {prompt}')
-        self.p = p
-        if seed is not None and int(seed) > 0:
-            self.seed = seed
-        elif hasattr(p, 'all_seeds'):
-            self.seed = p.all_seeds[0]
-        else:
-            self.seed = 0
-        self.prompt = prompt
-        self.image = image
-        if not grid:
-            self.batch_number = NOTHING if self.p is None or getattr(self.p, 'batch_size', 1) == 1 else (self.p.batch_index + 1 if hasattr(self.p, 'batch_index') else NOTHING)
-            self.iter_number = NOTHING if self.p is None or getattr(self.p, 'n_iter', 1) == 1 else (self.p.iteration + 1 if hasattr(self.p, 'iteration') else NOTHING)
-        else:
-            self.batch_number = NOTHING
-            self.iter_number = NOTHING
-
-    def hasprompt(self, *args):
-        lower = self.prompt.lower()
-        if getattr(self, 'p', None) is None or getattr(self, 'prompt', None) is None:
-            return None
-        outres = ""
-        for arg in args:
-            if arg != "":
-                division = arg.split("|")
-                expected = division[0].lower()
-                default = division[1] if len(division) > 1 else ""
-                if lower.find(expected) >= 0:
-                    outres = f'{outres}{expected}'
-                else:
-                    outres = outres if default == "" else f'{outres}{default}'
-        return outres
-
-    def image_hash(self):
-        if getattr(self, 'image', None) is None:
-            return None
-        import base64
-        from io import BytesIO
-        buffered = BytesIO()
-        self.image.save(buffered, format="JPEG")
-        img_str = base64.b64encode(buffered.getvalue())
-        shorthash = hashlib.sha256(img_str).hexdigest()[0:8]
-        return shorthash
-
-    def prompt_full(self):
-        return self.prompt_sanitize(self.prompt)
-
-    def prompt_words(self):
-        if getattr(self, 'prompt', None) is None:
-            return ''
-        no_attention = re_attention.sub(r'\1', self.prompt)
-        no_network = re_network.sub(r'\1', no_attention)
-        no_brackets = re_brackets.sub('', no_network)
-        words = [x for x in re_nonletters.split(no_brackets or "") if len(x) > 0]
-        prompt = " ".join(words[0:shared.opts.directories_max_prompt_words])
-        return self.prompt_sanitize(prompt)
-
-    def prompt_no_style(self):
-        if getattr(self, 'p', None) is None or getattr(self, 'prompt', None) is None:
-            return None
-        prompt_no_style = self.prompt
-        for style in shared.prompt_styles.get_style_prompts(self.p.styles):
-            if len(style) > 0:
-                for part in style.split("{prompt}"):
-                    prompt_no_style = prompt_no_style.replace(part, "").replace(", ,", ",")
-                prompt_no_style = prompt_no_style.replace(style, "")
-        return self.prompt_sanitize(prompt_no_style)
-
-    def datetime(self, *args):
-        import pytz
-        time_datetime = datetime.datetime.now()
-        time_format = args[0] if len(args) > 0 and args[0] != "" else self.default_time_format
-        try:
-            time_zone = pytz.timezone(args[1]) if len(args) > 1 else None
-        except pytz.exceptions.UnknownTimeZoneError:
-            time_zone = None
-        time_zone_time = time_datetime.astimezone(time_zone)
-        try:
-            formatted_time = time_zone_time.strftime(time_format)
-        except (ValueError, TypeError):
-            formatted_time = time_zone_time.strftime(self.default_time_format)
-        return formatted_time
-
-    def prompt_sanitize(self, prompt):
-        invalid_chars = '#<>:\'"\\|?*\n\t\r'
-        sanitized = prompt.translate({ ord(x): '_' for x in invalid_chars }).strip()
-        debug(f'Prompt sanitize: input="{prompt}" output={sanitized}')
-        return sanitized
-
-    def sanitize(self, filename):
-        invalid_chars = '\'"|?*\n\t\r' # <https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file>
-        invalid_folder = ':'
-        invalid_files = ['CON', 'PRN', 'AUX', 'NUL', 'NULL', 'COM0', 'COM1', 'LPT0', 'LPT1']
-        invalid_prefix = ', '
-        invalid_suffix = '.,_ '
-        fn, ext = os.path.splitext(filename)
-        parts = Path(fn).parts
-        newparts = []
-        for i, part in enumerate(parts):
-            part = part.translate({ ord(x): '_' for x in invalid_chars })
-            if i > 0 or (len(part) >= 2 and part[1] != invalid_folder): # skip drive, otherwise remove
-                part = part.translate({ ord(x): '_' for x in invalid_folder })
-            part = part.lstrip(invalid_prefix).rstrip(invalid_suffix)
-            if part in invalid_files: # reserved names
-                [part := part.replace(word, '_') for word in invalid_files] # pylint: disable=expression-not-assigned
-            newparts.append(part)
-        fn = str(Path(*newparts))
-        max_length = max(256 - len(ext), os.statvfs(__file__).f_namemax - 32 if hasattr(os, 'statvfs') else 256 - len(ext))
-        while len(os.path.abspath(fn)) > max_length:
-            fn = fn[:-1]
-        fn += ext
-        debug(f'Filename sanitize: input="{filename}" parts={parts} output="{fn}" ext={ext} max={max_length} len={len(fn)}')
-        return fn
-
-    def sequence(self, x, dirname, basename):
-        if shared.opts.save_images_add_number or '[seq]' in x:
-            if '[seq]' not in x:
-                x = os.path.join(os.path.dirname(x), f"[seq]-{os.path.basename(x)}")
-            basecount = get_next_sequence_number(dirname, basename)
-            for i in range(9999):
-                seq = f"{basecount + i:05}" if basename == '' else f"{basename}-{basecount + i:04}"
-                filename = x.replace('[seq]', seq)
-                if not os.path.exists(filename):
-                    debug(f'Prompt sequence: input="{x}" seq={seq} output="{filename}"')
-                    x = filename
-                    break
-        return x
-
-    def apply(self, x):
-        res = ''
-        for m in re_pattern.finditer(x):
-            text, pattern = m.groups()
-            if pattern is None:
-                res += text
-                continue
-            pattern_args = []
-            while True:
-                m = re_pattern_arg.match(pattern)
-                if m is None:
-                    break
-                pattern, arg = m.groups()
-                pattern_args.insert(0, arg)
-            fun = self.replacements.get(pattern.lower(), None)
-            if fun is not None:
-                try:
-                    debug(f'Filename apply: pattern={pattern.lower()} args={pattern_args}')
-                    replacement = fun(self, *pattern_args)
-                except Exception as e:
-                    replacement = None
-                    shared.log.error(f'Filename apply pattern: {x} {e}')
-                if replacement == NOTHING:
-                    continue
-                if replacement is not None:
-                    res += text + str(replacement).replace('/', '-').replace('\\', '-')
-                    continue
-            else:
-                res += text + f'[{pattern}]' # reinsert unknown pattern
-        return res
-
-
-def get_next_sequence_number(path, basename):
-    """
-    Determines and returns the next sequence number to use when saving an image in the specified directory.
-    """
-    result = -1
-    if basename != '':
-        basename = f"{basename}-"
-    prefix_length = len(basename)
-    if not os.path.isdir(path):
-        return 0
-    for p in os.listdir(path):
-        if p.startswith(basename):
-            parts = os.path.splitext(p[prefix_length:])[0].split('-')  # splits the filename (removing the basename first if one is defined, so the sequence number is always the first element)
-            try:
-                result = max(int(parts[0]), result)
-            except ValueError:
-                pass
-    return result + 1
-
-
 def atomically_save_image():
    Image.MAX_IMAGE_PIXELS = None # disable check in Pillow and rely on check below to allow large custom image sizes
    while True:
@@ -649,8 +101,25 @@ save_thread = threading.Thread(target=atomically_save_image, daemon=True)
 save_thread.start()


-def save_image(image, path, basename='', seed=None, prompt=None, extension=shared.opts.samples_format, info=None, short_filename=False, no_prompt=False, grid=False, pnginfo_section_name='parameters', p=None, existing_info=None, forced_filename=None, suffix='', save_to_dirs=None): # pylint: disable=unused-argument
-    debug(f'Save: fn={sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access
+def save_image(image,
+               path=None,
+               basename='',
+               seed=None,
+               prompt=None,
+               extension=shared.opts.samples_format,
+               info=None,
+               short_filename=False,
+               no_prompt=False,
+               grid=False,
+               pnginfo_section_name='parameters',
+               p=None,
+               existing_info=None,
+               forced_filename=None,
+               suffix='',
+               save_to_dirs=None,
+            ): # pylint: disable=unused-argument
+    fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
+    debug(f'Save: fn={fn}') # pylint: disable=protected-access
    if image is None:
        shared.log.warning('Image is none')
        return None, None, None
@@ -660,7 +129,7 @@ def save_image(image, path, basename='', seed=None, prompt=None, extension=share
        path = shared.opts.outdir_save
    namegen = FilenameGenerator(p, seed, prompt, image, grid=grid)
    suffix = suffix if suffix is not None else ''
-    basename = basename if basename is not None else ''
+    basename = '' if basename is None else basename
    if shared.opts.save_to_dirs:
        dirname = namegen.apply(shared.opts.directories_filename_pattern or "[prompt_words]")
        path = os.path.join(path, dirname)
@@ -671,9 +140,13 @@ def save_image(image, path, basename='', seed=None, prompt=None, extension=share
            file_decoration = "[seq]-[prompt_words]"
        file_decoration = namegen.apply(file_decoration)
        file_decoration += suffix if suffix is not None else ''
+        if file_decoration.startswith(basename):
+            basename = ''
        filename = os.path.join(path, f"{file_decoration}.{extension}") if basename == '' else os.path.join(path, f"{basename}-{file_decoration}.{extension}")
    else:
        forced_filename += suffix if suffix is not None else ''
+        if forced_filename.startswith(basename):
+            basename = ''
        filename = os.path.join(path, f"{forced_filename}.{extension}") if basename == '' else os.path.join(path, f"{basename}-{forced_filename}.{extension}")
    pnginfo = existing_info or {}
    if info is None:
@@ -0,0 +1,196 @@
+import math
+from collections import namedtuple
+import numpy as np
+from PIL import Image, ImageFont, ImageDraw
+from modules import shared, script_callbacks
+
+
+Grid = namedtuple("Grid", ["tiles", "tile_w", "tile_h", "image_w", "image_h", "overlap"])
+
+
+def check_grid_size(imgs):
+    mp = 0
+    for img in imgs:
+        mp += img.width * img.height if img is not None else 0
+    mp = round(mp / 1000000)
+    ok = mp <= shared.opts.img_max_size_mp
+    if not ok:
+        shared.log.warning(f'Maximum image size exceded: size={mp} maximum={shared.opts.img_max_size_mp} MPixels')
+    return ok
+
+
+def get_grid_size(imgs, batch_size=1, rows=None):
+    if rows is None:
+        if shared.opts.n_rows > 0:
+            rows = shared.opts.n_rows
+        elif shared.opts.n_rows == 0:
+            rows = batch_size
+        else:
+            rows = math.floor(math.sqrt(len(imgs)))
+            while len(imgs) % rows != 0:
+                rows -= 1
+    if rows > len(imgs):
+        rows = len(imgs)
+    cols = math.ceil(len(imgs) / rows)
+    return rows, cols
+
+
+def image_grid(imgs, batch_size=1, rows=None):
+    rows, cols = get_grid_size(imgs, batch_size, rows=rows)
+    params = script_callbacks.ImageGridLoopParams(imgs, cols, rows)
+    script_callbacks.image_grid_callback(params)
+    imgs = [i for i in imgs if i is not None] if imgs is not None else []
+    if len(imgs) == 0:
+        return None
+    w, h = max(i.width for i in imgs), max(i.height for i in imgs)
+    grid = Image.new('RGB', size=(params.cols * w, params.rows * h), color=shared.opts.grid_background)
+    for i, img in enumerate(params.imgs):
+        grid.paste(img, box=(i % params.cols * w, i // params.cols * h))
+    return grid
+
+
+def split_grid(image, tile_w=512, tile_h=512, overlap=64):
+    w = image.width
+    h = image.height
+    non_overlap_width = tile_w - overlap
+    non_overlap_height = tile_h - overlap
+    cols = math.ceil((w - overlap) / non_overlap_width)
+    rows = math.ceil((h - overlap) / non_overlap_height)
+    dx = (w - tile_w) / (cols - 1) if cols > 1 else 0
+    dy = (h - tile_h) / (rows - 1) if rows > 1 else 0
+    grid = Grid([], tile_w, tile_h, w, h, overlap)
+    for row in range(rows):
+        row_images = []
+        y = int(row * dy)
+        if y + tile_h >= h:
+            y = h - tile_h
+        for col in range(cols):
+            x = int(col * dx)
+            if x + tile_w >= w:
+                x = w - tile_w
+            tile = image.crop((x, y, x + tile_w, y + tile_h))
+            row_images.append([x, tile_w, tile])
+        grid.tiles.append([y, tile_h, row_images])
+    return grid
+
+
+def combine_grid(grid):
+    def make_mask_image(r):
+        r = r * 255 / grid.overlap
+        r = r.astype(np.uint8)
+        return Image.fromarray(r, 'L')
+
+    mask_w = make_mask_image(np.arange(grid.overlap, dtype=np.float32).reshape((1, grid.overlap)).repeat(grid.tile_h, axis=0))
+    mask_h = make_mask_image(np.arange(grid.overlap, dtype=np.float32).reshape((grid.overlap, 1)).repeat(grid.image_w, axis=1))
+    combined_image = Image.new("RGB", (grid.image_w, grid.image_h))
+    for y, h, row in grid.tiles:
+        combined_row = Image.new("RGB", (grid.image_w, h))
+        for x, w, tile in row:
+            if x == 0:
+                combined_row.paste(tile, (0, 0))
+                continue
+            combined_row.paste(tile.crop((0, 0, grid.overlap, h)), (x, 0), mask=mask_w)
+            combined_row.paste(tile.crop((grid.overlap, 0, w, h)), (x + grid.overlap, 0))
+        if y == 0:
+            combined_image.paste(combined_row, (0, 0))
+            continue
+        combined_image.paste(combined_row.crop((0, 0, combined_row.width, grid.overlap)), (0, y), mask=mask_h)
+        combined_image.paste(combined_row.crop((0, grid.overlap, combined_row.width, h)), (0, y + grid.overlap))
+    return combined_image
+
+
+class GridAnnotation:
+    def __init__(self, text='', is_active=True):
+        self.text = text
+        self.is_active = is_active
+        self.size = None
+
+
+def get_font(fontsize):
+    try:
+        return ImageFont.truetype(shared.opts.font or "javascript/notosans-nerdfont-regular.ttf", fontsize)
+    except Exception:
+        return ImageFont.truetype("javascript/notosans-nerdfont-regular.ttf", fontsize)
+
+
+def draw_grid_annotations(im, width, height, hor_texts, ver_texts, margin=0, title=None):
+    def wrap(drawing, text, font, line_length):
+        lines = ['']
+        for word in text.split():
+            line = f'{lines[-1]} {word}'.strip()
+            if drawing.textlength(line, font=font) <= line_length:
+                lines[-1] = line
+            else:
+                lines.append(word)
+        return lines
+
+    def draw_texts(drawing: ImageDraw, draw_x, draw_y, lines, initial_fnt, initial_fontsize):
+        for line in lines:
+            font = initial_fnt
+            fontsize = initial_fontsize
+            while drawing.multiline_textbbox((0,0), text=line.text, font=font)[2] > line.allowed_width and fontsize > 0:
+                fontsize -= 1
+                font = get_font(fontsize)
+            drawing.multiline_text((draw_x, draw_y + line.size[1] / 2), line.text, font=font, fill=shared.opts.font_color if line.is_active else color_inactive, anchor="mm", align="center")
+            if not line.is_active:
+                drawing.line((draw_x - line.size[0] // 2, draw_y + line.size[1] // 2, draw_x + line.size[0] // 2, draw_y + line.size[1] // 2), fill=color_inactive, width=4)
+            draw_y += line.size[1] + line_spacing
+
+    fontsize = (width + height) // 25
+    line_spacing = fontsize // 2
+    font = get_font(fontsize)
+    color_inactive = (127, 127, 127)
+    pad_left = 0 if sum([sum([len(line.text) for line in lines]) for lines in ver_texts]) == 0 else width * 3 // 4
+    cols = len(hor_texts)
+    rows = len(ver_texts)
+    # assert cols == len(hor_texts), f'bad number of horizontal texts: {len(hor_texts)}; must be {cols}'
+    # assert rows == len(hor_texts), f'bad number of vertical texts: {len(ver_texts)}; must be {rows}'
+    calc_img = Image.new("RGB", (1, 1), shared.opts.grid_background)
+    calc_d = ImageDraw.Draw(calc_img)
+    title_texts = [title] if title else [[GridAnnotation()]]
+    for texts, allowed_width in zip(hor_texts + ver_texts + title_texts, [width] * len(hor_texts) + [pad_left] * len(ver_texts) + [(width+margin)*cols]):
+        items = [] + texts
+        texts.clear()
+        for line in items:
+            wrapped = wrap(calc_d, line.text, font, allowed_width)
+            texts += [GridAnnotation(x, line.is_active) for x in wrapped]
+        for line in texts:
+            bbox = calc_d.multiline_textbbox((0, 0), line.text, font=font)
+            line.size = (bbox[2] - bbox[0], bbox[3] - bbox[1])
+            line.allowed_width = allowed_width
+    hor_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing for lines in hor_texts]
+    ver_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing * len(lines) for lines in ver_texts]
+    pad_top = 0 if sum(hor_text_heights) == 0 else max(hor_text_heights) + line_spacing * 2
+    title_pad = 0
+    if title:
+        title_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing for lines in title_texts] # pylint: disable=unsubscriptable-object
+        title_pad = 0 if sum(title_text_heights) == 0 else max(title_text_heights) + line_spacing * 2
+    result = Image.new("RGB", (im.width + pad_left + margin * (cols-1), im.height + pad_top + title_pad + margin * (rows-1)), shared.opts.grid_background)
+    for row in range(rows):
+        for col in range(cols):
+            cell = im.crop((width * col, height * row, width * (col+1), height * (row+1)))
+            result.paste(cell, (pad_left + (width + margin) * col, pad_top + title_pad + (height + margin) * row))
+    d = ImageDraw.Draw(result)
+    if title:
+        x = pad_left + ((width+margin)*cols) / 2
+        y = title_pad / 2 - title_text_heights[0] / 2
+        draw_texts(d, x, y, title_texts[0], font, fontsize)
+    for col in range(cols):
+        x = pad_left + (width + margin) * col + width / 2
+        y = (pad_top / 2 - hor_text_heights[col] / 2) + title_pad
+        draw_texts(d, x, y, hor_texts[col], font, fontsize)
+    for row in range(rows):
+        x = pad_left / 2
+        y = (pad_top + (height + margin) * row + height / 2 - ver_text_heights[row] / 2) + title_pad
+        draw_texts(d, x, y, ver_texts[row], font, fontsize)
+    return result
+
+
+def draw_prompt_matrix(im, width, height, all_prompts, margin=0):
+    prompts = all_prompts[1:]
+    boundary = math.ceil(len(prompts) / 2)
+    prompts_horiz = prompts[:boundary]
+    prompts_vert = prompts[boundary:]
+    hor_texts = [[GridAnnotation(x, is_active=pos & (1 << i) != 0) for i, x in enumerate(prompts_horiz)] for pos in range(1 << len(prompts_horiz))]
+    ver_texts = [[GridAnnotation(x, is_active=pos & (1 << i) != 0) for i, x in enumerate(prompts_vert)] for pos in range(1 << len(prompts_vert))]
+    return draw_grid_annotations(im, width, height, hor_texts, ver_texts, margin)
@@ -0,0 +1,243 @@
+import re
+import os
+import uuid
+import string
+import hashlib
+import datetime
+from pathlib import Path
+from modules import shared, errors
+
+
+debug = errors.log.trace if os.environ.get('SD_NAMEGEN_DEBUG', None) is not None else lambda *args, **kwargs: None
+re_nonletters = re.compile(r'[\s' + string.punctuation + ']+')
+re_pattern = re.compile(r"(.*?)(?:\[([^\[\]]+)\]|$)")
+re_pattern_arg = re.compile(r"(.*)<([^>]*)>$")
+re_attention = re.compile(r'[\(*\[*](\w+)(:\d+(\.\d+))?[\)*\]*]|')
+re_network = re.compile(r'\<\w+:(\w+)(:\d+(\.\d+))?\>|')
+re_brackets = re.compile(r'[\([{})\]]')
+NOTHING = object()
+
+
+class FilenameGenerator:
+    replacements = {
+        'width': lambda self: self.image.width,
+        'height': lambda self: self.image.height,
+        'batch_number': lambda self: self.batch_number,
+        'iter_number': lambda self: self.iter_number,
+        'num': lambda self: NOTHING if self.p.n_iter == 1 and self.p.batch_size == 1 else self.p.iteration * self.p.batch_size + self.p.batch_index + 1,
+        'generation_number': lambda self: NOTHING if self.p.n_iter == 1 and self.p.batch_size == 1 else self.p.iteration * self.p.batch_size + self.p.batch_index + 1,
+        'date': lambda self: datetime.datetime.now().strftime('%Y-%m-%d'),
+        'datetime': lambda self, *args: self.datetime(*args),  # accepts formats: [datetime], [datetime<Format>], [datetime<Format><Time Zone>]
+        'hasprompt': lambda self, *args: self.hasprompt(*args),  # accepts formats:[hasprompt<prompt1|default><prompt2>..]
+        'hash': lambda self: self.image_hash(),
+        'image_hash': lambda self: self.image_hash(),
+        'timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp),
+        'job_timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp),
+
+        'model': lambda self: shared.sd_model.sd_checkpoint_info.title if shared.sd_loaded else '',
+        'model_shortname': lambda self: shared.sd_model.sd_checkpoint_info.model_name if shared.sd_loaded else '',
+        'model_name': lambda self: shared.sd_model.sd_checkpoint_info.model_name if shared.sd_loaded else '',
+        'model_hash': lambda self: shared.sd_model.sd_checkpoint_info.shorthash if shared.sd_loaded else '',
+
+        'prompt': lambda self: self.prompt_full(),
+        'prompt_no_styles': lambda self: self.prompt_no_style(),
+        'prompt_words': lambda self: self.prompt_words(),
+        'prompt_hash': lambda self: hashlib.sha256(self.prompt.encode()).hexdigest()[0:8],
+
+        'sampler': lambda self: self.p and self.p.sampler_name,
+        'seed': lambda self: self.seed and str(self.seed) or '',
+        'steps': lambda self: self.p and getattr(self.p, 'steps', 0),
+        'cfg': lambda self: self.p and getattr(self.p, 'cfg_scale', 0),
+        'clip_skip': lambda self: self.p and getattr(self.p, 'clip_skip', 0),
+        'denoising': lambda self: self.p and getattr(self.p, 'denoising_strength', 0),
+        'styles': lambda self: self.p and ", ".join([style for style in self.p.styles if not style == "None"]) or "None",
+        'uuid': lambda self: str(uuid.uuid4()),
+    }
+    default_time_format = '%Y%m%d%H%M%S'
+
+    def __init__(self, p, seed, prompt, image, grid=False):
+        if p is None:
+            debug('Filename generator init skip')
+        else:
+            debug(f'Filename generator init: {seed} {prompt}')
+        self.p = p
+        if seed is not None and int(seed) > 0:
+            self.seed = seed
+        elif p is not None and hasattr(p, 'all_seeds'):
+            self.seed = p.all_seeds[0]
+        else:
+            self.seed = p.seed if p is not None else 0
+        if prompt is not None:
+            self.prompt = prompt
+        else:
+            self.prompt = p.prompt if p is not None else ''
+        self.image = image
+        if not grid:
+            self.batch_number = NOTHING if self.p is None or getattr(self.p, 'batch_size', 1) == 1 else (self.p.batch_index + 1 if hasattr(self.p, 'batch_index') else NOTHING)
+            self.iter_number = NOTHING if self.p is None or getattr(self.p, 'n_iter', 1) == 1 else (self.p.iteration + 1 if hasattr(self.p, 'iteration') else NOTHING)
+        else:
+            self.batch_number = NOTHING
+            self.iter_number = NOTHING
+
+    def hasprompt(self, *args):
+        lower = self.prompt.lower()
+        if getattr(self, 'p', None) is None or getattr(self, 'prompt', None) is None:
+            return None
+        outres = ""
+        for arg in args:
+            if arg != "":
+                division = arg.split("|")
+                expected = division[0].lower()
+                default = division[1] if len(division) > 1 else ""
+                if lower.find(expected) >= 0:
+                    outres = f'{outres}{expected}'
+                else:
+                    outres = outres if default == "" else f'{outres}{default}'
+        return outres
+
+    def image_hash(self):
+        if getattr(self, 'image', None) is None:
+            return None
+        import base64
+        from io import BytesIO
+        buffered = BytesIO()
+        self.image.save(buffered, format="JPEG")
+        img_str = base64.b64encode(buffered.getvalue())
+        shorthash = hashlib.sha256(img_str).hexdigest()[0:8]
+        return shorthash
+
+    def prompt_full(self):
+        return self.prompt_sanitize(self.prompt)
+
+    def prompt_words(self):
+        if getattr(self, 'prompt', None) is None:
+            return ''
+        no_attention = re_attention.sub(r'\1', self.prompt)
+        no_network = re_network.sub(r'\1', no_attention)
+        no_brackets = re_brackets.sub('', no_network)
+        words = [x for x in re_nonletters.split(no_brackets or "") if len(x) > 0]
+        prompt = " ".join(words[0:shared.opts.directories_max_prompt_words])
+        return self.prompt_sanitize(prompt)
+
+    def prompt_no_style(self):
+        if getattr(self, 'p', None) is None or getattr(self, 'prompt', None) is None:
+            return None
+        prompt_no_style = self.prompt
+        for style in shared.prompt_styles.get_style_prompts(self.p.styles):
+            if len(style) > 0:
+                for part in style.split("{prompt}"):
+                    prompt_no_style = prompt_no_style.replace(part, "").replace(", ,", ",")
+                prompt_no_style = prompt_no_style.replace(style, "")
+        return self.prompt_sanitize(prompt_no_style)
+
+    def datetime(self, *args):
+        import pytz
+        time_datetime = datetime.datetime.now()
+        time_format = args[0] if len(args) > 0 and args[0] != "" else self.default_time_format
+        try:
+            time_zone = pytz.timezone(args[1]) if len(args) > 1 else None
+        except pytz.exceptions.UnknownTimeZoneError:
+            time_zone = None
+        time_zone_time = time_datetime.astimezone(time_zone)
+        try:
+            formatted_time = time_zone_time.strftime(time_format)
+        except (ValueError, TypeError):
+            formatted_time = time_zone_time.strftime(self.default_time_format)
+        return formatted_time
+
+    def prompt_sanitize(self, prompt):
+        invalid_chars = '#<>:\'"\\|?*\n\t\r'
+        sanitized = prompt.translate({ ord(x): '_' for x in invalid_chars }).strip()
+        debug(f'Prompt sanitize: input="{prompt}" output={sanitized}')
+        return sanitized
+
+    def sanitize(self, filename):
+        invalid_chars = '\'"|?*\n\t\r' # <https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file>
+        invalid_folder = ':'
+        invalid_files = ['CON', 'PRN', 'AUX', 'NUL', 'NULL', 'COM0', 'COM1', 'LPT0', 'LPT1']
+        invalid_prefix = ', '
+        invalid_suffix = '.,_ '
+        fn, ext = os.path.splitext(filename)
+        parts = Path(fn).parts
+        newparts = []
+        for i, part in enumerate(parts):
+            part = part.translate({ ord(x): '_' for x in invalid_chars })
+            if i > 0 or (len(part) >= 2 and part[1] != invalid_folder): # skip drive, otherwise remove
+                part = part.translate({ ord(x): '_' for x in invalid_folder })
+            part = part.lstrip(invalid_prefix).rstrip(invalid_suffix)
+            if part in invalid_files: # reserved names
+                [part := part.replace(word, '_') for word in invalid_files] # pylint: disable=expression-not-assigned
+            newparts.append(part)
+        fn = str(Path(*newparts))
+        max_length = max(256 - len(ext), os.statvfs(__file__).f_namemax - 32 if hasattr(os, 'statvfs') else 256 - len(ext))
+        while len(os.path.abspath(fn)) > max_length:
+            fn = fn[:-1]
+        fn += ext
+        debug(f'Filename sanitize: input="{filename}" parts={parts} output="{fn}" ext={ext} max={max_length} len={len(fn)}')
+        return fn
+
+    def sequence(self, fn, dirname, basename):
+        x = fn
+        if shared.opts.save_images_add_number or '[seq]' in fn:
+            if '[seq]' not in fn:
+                fn = os.path.join(os.path.dirname(fn), f"[seq]-{os.path.basename(fn)}")
+            basecount = get_next_sequence_number(dirname, basename)
+            for i in range(9999):
+                seq = f"{basecount + i:05}"
+                filename = fn.replace('[seq]', seq)
+                if not os.path.exists(filename):
+                    debug(f'Prompt sequence: input="{fn}" seq={seq} output="{filename}"')
+                    x = filename
+                    break
+        return x
+
+    def apply(self, x):
+        res = ''
+        for m in re_pattern.finditer(x):
+            text, pattern = m.groups()
+            if pattern is None:
+                res += text
+                continue
+            pattern_args = []
+            while True:
+                m = re_pattern_arg.match(pattern)
+                if m is None:
+                    break
+                pattern, arg = m.groups()
+                pattern_args.insert(0, arg)
+            fun = self.replacements.get(pattern.lower(), None)
+            if fun is not None:
+                try:
+                    debug(f'Filename apply: pattern={pattern.lower()} args={pattern_args}')
+                    replacement = fun(self, *pattern_args)
+                except Exception as e:
+                    replacement = None
+                    shared.log.error(f'Filename apply pattern: {x} {e}')
+                if replacement == NOTHING:
+                    continue
+                if replacement is not None:
+                    res += text + str(replacement).replace('/', '-').replace('\\', '-')
+                    continue
+            else:
+                res += text + f'[{pattern}]' # reinsert unknown pattern
+        return res
+
+
+def get_next_sequence_number(path, basename):
+    """
+    Determines and returns the next sequence number to use when saving an image in the specified directory.
+    """
+    result = -1
+    if basename != '':
+        basename = f"{basename}-"
+    prefix_length = len(basename)
+    if not os.path.isdir(path):
+        return 0
+    for p in os.listdir(path):
+        if p.startswith(basename):
+            parts = os.path.splitext(p[prefix_length:])[0].split('-')  # splits the filename (removing the basename first if one is defined, so the sequence number is always the first element)
+            try:
+                result = max(int(parts[0]), result)
+            except ValueError:
+                pass
+    return result + 1
@@ -0,0 +1,133 @@
+import sys
+import time
+import numpy as np
+from PIL import Image
+from modules import shared
+
+
+def resize_image(resize_mode, im, width, height, upscaler_name=None, output_type='image', context=None):
+    upscaler_name = upscaler_name or shared.opts.upscaler_for_img2img
+
+    def latent(im, w, h, upscaler):
+        from modules.processing_vae import vae_encode, vae_decode
+        import torch
+        latents = vae_encode(im, shared.sd_model, full_quality=False) # TODO enable full VAE mode for resize-latent
+        latents = torch.nn.functional.interpolate(latents, size=(int(h // 8), int(w // 8)), mode=upscaler["mode"], antialias=upscaler["antialias"])
+        im = vae_decode(latents, shared.sd_model, output_type='pil', full_quality=False)[0]
+        return im
+
+    def resize(im, w, h):
+        w = int(w)
+        h = int(h)
+        if upscaler_name is None or upscaler_name == "None" or im.mode == 'L':
+            return im.resize((w, h), resample=Image.Resampling.LANCZOS) # force for mask
+        scale = max(w / im.width, h / im.height)
+        if scale > 1.0:
+            upscalers = [x for x in shared.sd_upscalers if x.name.lower().replace('-', ' ') == upscaler_name.lower().replace('-', ' ')]
+            if len(upscalers) > 0:
+                upscaler = upscalers[0]
+                im = upscaler.scaler.upscale(im, scale, upscaler.data_path)
+            else:
+                upscaler = shared.latent_upscale_modes.get(upscaler_name, None)
+                if upscaler is not None:
+                    im = latent(im, w, h, upscaler)
+                else:
+                    upscaler = shared.sd_upscalers[0]
+                    shared.log.warning(f"Resize upscaler: invalid={upscaler_name} fallback={upscaler.name}")
+                    shared.log.debug(f"Resize upscaler: available={[u.name for u in shared.sd_upscalers]}")
+        if im.width != w or im.height != h: # probably downsample after upscaler created larger image
+            im = im.resize((w, h), resample=Image.Resampling.LANCZOS)
+        return im
+
+    def crop(im):
+        ratio = width / height
+        src_ratio = im.width / im.height
+        src_w = width if ratio > src_ratio else im.width * height // im.height
+        src_h = height if ratio <= src_ratio else im.height * width // im.width
+        resized = resize(im, src_w, src_h)
+        res = Image.new(im.mode, (width, height))
+        res.paste(resized, box=(width // 2 - src_w // 2, height // 2 - src_h // 2))
+        return res
+
+    def fill(im, color=None):
+        color = color or shared.opts.image_background
+        """
+        ratio = round(width / height, 1)
+        src_ratio = round(im.width / im.height, 1)
+        src_w = width if ratio < src_ratio else im.width * height // im.height
+        src_h = height if ratio >= src_ratio else im.height * width // im.width
+        resized = resize(im, src_w, src_h)
+        res = Image.new(im.mode, (width, height))
+        res.paste(resized, box=(width // 2 - src_w // 2, height // 2 - src_h // 2))
+        if ratio < src_ratio:
+            fill_height = height // 2 - src_h // 2
+            if width > 0 and fill_height > 0:
+                res.paste(resized.resize((width, fill_height), box=(0, 0, width, 0)), box=(0, 0))
+                res.paste(resized.resize((width, fill_height), box=(0, resized.height, width, resized.height)), box=(0, fill_height + src_h))
+        elif ratio > src_ratio:
+            fill_width = width // 2 - src_w // 2
+            if height > 0 and fill_width > 0:
+                res.paste(resized.resize((fill_width, height), box=(0, 0, 0, height)), box=(0, 0))
+                res.paste(resized.resize((fill_width, height), box=(resized.width, 0, resized.width, height)), box=(fill_width + src_w, 0))
+        return res
+        """
+        ratio = min(width / im.width, height / im.height)
+        im = resize(im, int(im.width * ratio), int(im.height * ratio))
+        res = Image.new(im.mode, (width, height), color=color)
+        res.paste(im, box=((width - im.width)//2, (height - im.height)//2))
+        return res
+
+    def context_aware(im, width, height, context):
+        import seam_carving # https://github.com/li-plus/seam-carving
+        if 'forward' in context:
+            energy_mode = "forward"
+        elif 'backward' in context:
+            energy_mode = "backward"
+        else:
+            return im
+        if 'Add' in context:
+            src_ratio = min(width / im.width, height / im.height)
+            src_w = int(im.width * src_ratio)
+            src_h = int(im.height * src_ratio)
+            src_image = resize(im, src_w, src_h)
+        elif 'Remove' in context:
+            ratio = width / height
+            src_ratio = im.width / im.height
+            src_w = width if ratio > src_ratio else im.width * height // im.height
+            src_h = height if ratio <= src_ratio else im.height * width // im.width
+            src_image = resize(im, src_w, src_h)
+        else:
+            return im
+        res = Image.fromarray(seam_carving.resize(
+            src_image, # source image (rgb or gray)
+            size=(width, height),  # target size
+            energy_mode=energy_mode,  # choose from {backward, forward}
+            order="width-first",  # choose from {width-first, height-first}
+            keep_mask=None,  # object mask to protect from removal
+        ))
+        return res
+
+    t0 = time.time()
+    if resize_mode is None:
+        resize_mode = 0
+    if resize_mode == 0 or (im.width == width and im.height == height) or (width == 0 and height == 0): # none
+        res = im.copy()
+    elif resize_mode == 1: # fixed
+        res = resize(im, width, height)
+    elif resize_mode == 2: # crop
+        res = crop(im)
+    elif resize_mode == 3: # fill
+        res = fill(im)
+    elif resize_mode == 4: # edge
+        from modules import masking
+        res = fill(im, color=0)
+        res, _mask = masking.outpaint(res)
+    elif resize_mode == 5:  # context-aware
+        res = context_aware(im, width, height, context)
+    else:
+        res = im.copy()
+        shared.log.error(f'Invalid resize mode: {resize_mode}')
+    t1 = time.time()
+    fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
+    shared.log.debug(f'Image resize: input={im} width={width} height={height} mode="{shared.resize_modes[resize_mode]}" upscaler="{upscaler_name}" context="{context}" type={output_type} result={res} time={t1-t0:.2f} fn={fn}') # pylint: disable=protected-access
+    return np.array(res) if output_type == 'np' else res
@@ -107,7 +107,7 @@ def process_batch(p, input_files, input_dir, output_dir, inpaint_mask_dir, args)
        shared.log.debug(f'Processed: images={len(batch_image_files)} memory={memory_stats()} batch')


-def img2img(id_task: str, mode: int,
+def img2img(id_task: str, state: str, mode: int,
            prompt, negative_prompt, prompt_styles,
            init_img,
            sketch,
@@ -120,7 +120,7 @@ def img2img(id_task: str, mode: int,
            sampler_index,
            mask_blur, mask_alpha,
            inpainting_fill,
-            full_quality, restore_faces, tiling, hidiffusion,
+            full_quality, detailer, tiling, hidiffusion,
            n_iter, batch_size,
            cfg_scale, image_cfg_scale,
            diffusers_guidance_rescale,
@@ -144,7 +144,7 @@ def img2img(id_task: str, mode: int,
        shared.log.warning('Model not loaded')
        return [], '', '', 'Error: model not loaded'

-    debug(f'img2img: id_task={id_task}|mode={mode}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|init_img={init_img}|sketch={sketch}|init_img_with_mask={init_img_with_mask}|inpaint_color_sketch={inpaint_color_sketch}|inpaint_color_sketch_orig={inpaint_color_sketch_orig}|init_img_inpaint={init_img_inpaint}|init_mask_inpaint={init_mask_inpaint}|steps={steps}|sampler_index={sampler_index}||mask_blur={mask_blur}|mask_alpha={mask_alpha}|inpainting_fill={inpainting_fill}|full_quality={full_quality}|restore_faces={restore_faces}|tiling={tiling}|hidiffusion={hidiffusion}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|image_cfg_scale={image_cfg_scale}|clip_skip={clip_skip}|denoising_strength={denoising_strength}|seed={seed}|subseed{subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}|selected_scale_tab={selected_scale_tab}|height={height}|width={width}|scale_by={scale_by}|resize_mode={resize_mode}|resize_name={resize_name}|resize_context={resize_context}|inpaint_full_res={inpaint_full_res}|inpaint_full_res_padding={inpaint_full_res_padding}|inpainting_mask_invert={inpainting_mask_invert}|img2img_batch_files={img2img_batch_files}|img2img_batch_input_dir={img2img_batch_input_dir}|img2img_batch_output_dir={img2img_batch_output_dir}|img2img_batch_inpaint_mask_dir={img2img_batch_inpaint_mask_dir}|override_settings_texts={override_settings_texts}')
+    debug(f'img2img: id_task={id_task}|mode={mode}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|init_img={init_img}|sketch={sketch}|init_img_with_mask={init_img_with_mask}|inpaint_color_sketch={inpaint_color_sketch}|inpaint_color_sketch_orig={inpaint_color_sketch_orig}|init_img_inpaint={init_img_inpaint}|init_mask_inpaint={init_mask_inpaint}|steps={steps}|sampler_index={sampler_index}||mask_blur={mask_blur}|mask_alpha={mask_alpha}|inpainting_fill={inpainting_fill}|full_quality={full_quality}|detailer={detailer}|tiling={tiling}|hidiffusion={hidiffusion}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|image_cfg_scale={image_cfg_scale}|clip_skip={clip_skip}|denoising_strength={denoising_strength}|seed={seed}|subseed{subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}|selected_scale_tab={selected_scale_tab}|height={height}|width={width}|scale_by={scale_by}|resize_mode={resize_mode}|resize_name={resize_name}|resize_context={resize_context}|inpaint_full_res={inpaint_full_res}|inpaint_full_res_padding={inpaint_full_res_padding}|inpainting_mask_invert={inpainting_mask_invert}|img2img_batch_files={img2img_batch_files}|img2img_batch_input_dir={img2img_batch_input_dir}|img2img_batch_output_dir={img2img_batch_output_dir}|img2img_batch_inpaint_mask_dir={img2img_batch_inpaint_mask_dir}|override_settings_texts={override_settings_texts}')

    if mode == 5:
        if img2img_batch_files is None or len(img2img_batch_files) == 0:
@@ -225,7 +225,7 @@ def img2img(id_task: str, mode: int,
        width=width,
        height=height,
        full_quality=full_quality,
-        restore_faces=restore_faces,
+        detailer=detailer,
        tiling=tiling,
        hidiffusion=hidiffusion,
        init_images=[image],
@@ -251,6 +251,7 @@ def img2img(id_task: str, mode: int,
    )
    p.scripts = modules.scripts.scripts_img2img
    p.script_args = args
+    p.state = state
    if mask:
        p.extra_generation_params["Mask blur"] = mask_blur
        p.extra_generation_params["Mask alpha"] = mask_alpha
@@ -3,7 +3,11 @@ import re
 import json


-debug = lambda *args, **kwargs: None # pylint: disable=unnecessary-lambda-assignment
+if os.environ.get('SD_PASTE_DEBUG', None) is not None:
+    from modules.errors import log
+    debug = log.trace
+else:
+    debug = lambda *args, **kwargs: None # pylint: disable=unnecessary-lambda-assignment
 re_size = re.compile(r"^(\d+)x(\d+)$") # int x int
 re_param = re.compile(r'\s*([\w ]+):\s*("(?:\\"[^,]|\\"|\\|[^\"])+"|[^,]*)(?:,|$)') # multi-word: value

@@ -75,38 +79,34 @@ def parse(infotext):


 mapping = [
+    # Backend
    ('Backend', 'sd_backend'),
+    # Models
    ('Model hash', 'sd_model_checkpoint'),
    ('Refiner', 'sd_model_refiner'),
    ('VAE', 'sd_vae'),
+    ('TE', 'sd_text_encoder'),
+    ('Unet', 'sd_unet'),
+    # Other
    ('Parser', 'prompt_attention'),
    ('Color correction', 'img2img_color_correction'),
    # Samplers
-    ('Sampler Eta', 'scheduler_eta'),
-    ('Sampler ENSD', 'eta_noise_seed_delta'),
+    ('Sampler eta delta', 'eta_noise_seed_delta'),
+    ('Sampler eta multiplier', 'initial_noise_multiplier'),
+    ('Sampler timesteps', 'schedulers_timesteps'),
+    ('Sampler spacing', 'schedulers_timestep_spacing'),
+    ('Sampler sigma', 'schedulers_sigma'),
    ('Sampler order', 'schedulers_solver_order'),
-    # Samplers diffusers
+    ('Sampler type', 'schedulers_prediction_type'),
    ('Sampler beta schedule', 'schedulers_beta_schedule'),
+    ('Sampler low order', 'schedulers_use_loworder'),
+    ('Sampler dynamic', 'schedulers_use_thresholding'),
+    ('Sampler rescale', 'schedulers_rescale_betas'),
    ('Sampler beta start', 'schedulers_beta_start'),
    ('Sampler beta end', 'schedulers_beta_end'),
-    ('Sampler DPM solver', 'schedulers_dpm_solver'),
-    # Samplers original
-    ('Sampler brownian', 'schedulers_brownian_noise'),
-    ('Sampler discard', 'schedulers_discard_penultimate'),
-    ('Sampler dyn threshold', 'schedulers_use_thresholding'),
-    ('Sampler karras', 'schedulers_use_karras'),
-    ('Sampler low order', 'schedulers_use_loworder'),
-    ('Sampler quantization', 'enable_quantization'),
-    ('Sampler sigma', 'schedulers_sigma'),
-    ('Sampler sigma min', 's_min'),
-    ('Sampler sigma max', 's_max'),
-    ('Sampler sigma churn', 's_churn'),
-    ('Sampler sigma uncond', 's_min_uncond'),
-    ('Sampler sigma noise', 's_noise'),
-    ('Sampler sigma tmin', 's_tmin'),
-    ('Sampler ENSM', 'initial_noise_multiplier'), # img2img only
-    ('UniPC skip type', 'uni_pc_skip_type'),
-    ('UniPC variant', 'uni_pc_variant'),
+    ('Sampler range', 'schedulers_timesteps_range'),
+    ('Sampler shift', 'schedulers_shift'),
+    ('Sampler dynamic shift', 'schedulers_dynamic_shift'),
    # Token Merging
    ('Mask weight', 'inpainting_mask_weight'),
    ('ToMe', 'tome_ratio'),
@@ -16,8 +16,13 @@ def ipex_init(): # pylint: disable=too-many-statements
        if hasattr(torch, "cuda") and hasattr(torch.cuda, "is_xpu_hijacked") and torch.cuda.is_xpu_hijacked:
            return True, "Skipping IPEX hijack"
        else:
-            device_supports_fp64 = torch.xpu.has_fp64_dtype() if hasattr(torch.xpu, "has_fp64_dtype") else torch.xpu.get_device_properties("xpu").has_fp64
-
+            try: # force xpu device on torch compile and triton
+                torch._inductor.utils.GPU_TYPES = ["xpu"]
+                torch._inductor.utils.get_gpu_type = lambda *args, **kwargs: "xpu"
+                from triton import backends as triton_backends # pylint: disable=import-error
+                triton_backends.backends["nvidia"].driver.is_active = lambda *args, **kwargs: False
+            except Exception:
+                pass
            # Replace cuda with xpu:
            torch.cuda.current_device = torch.xpu.current_device
            torch.cuda.current_stream = torch.xpu.current_stream
@@ -117,26 +122,26 @@ def ipex_init(): # pylint: disable=too-many-statements
                torch.cuda.traceback = torch.xpu.traceback

            # Memory:
-            if 'linux' in sys.platform and "WSL2" in os.popen("uname -a").read():
+            if legacy and 'linux' in sys.platform and "WSL2" in os.popen("uname -a").read():
                torch.xpu.empty_cache = lambda: None
            torch.cuda.empty_cache = torch.xpu.empty_cache

            if legacy:
-                torch.cuda.memory = torch.xpu.memory
-                torch.cuda.memory_stats = torch.xpu.memory_stats
                torch.cuda.memory_summary = torch.xpu.memory_summary
                torch.cuda.memory_snapshot = torch.xpu.memory_snapshot
-                torch.cuda.memory_allocated = torch.xpu.memory_allocated
-                torch.cuda.max_memory_allocated = torch.xpu.max_memory_allocated
-                torch.cuda.memory_reserved = torch.xpu.memory_reserved
-                torch.cuda.memory_cached = torch.xpu.memory_reserved
-                torch.cuda.max_memory_reserved = torch.xpu.max_memory_reserved
-                torch.cuda.max_memory_cached = torch.xpu.max_memory_reserved
-                torch.cuda.reset_peak_memory_stats = torch.xpu.reset_peak_memory_stats
-                torch.cuda.reset_max_memory_cached = torch.xpu.reset_peak_memory_stats
-                torch.cuda.reset_max_memory_allocated = torch.xpu.reset_peak_memory_stats
-                torch.cuda.memory_stats_as_nested_dict = torch.xpu.memory_stats_as_nested_dict
-                torch.cuda.reset_accumulated_memory_stats = torch.xpu.reset_accumulated_memory_stats
+            torch.cuda.memory = torch.xpu.memory
+            torch.cuda.memory_stats = torch.xpu.memory_stats
+            torch.cuda.memory_allocated = torch.xpu.memory_allocated
+            torch.cuda.max_memory_allocated = torch.xpu.max_memory_allocated
+            torch.cuda.memory_reserved = torch.xpu.memory_reserved
+            torch.cuda.memory_cached = torch.xpu.memory_reserved
+            torch.cuda.max_memory_reserved = torch.xpu.max_memory_reserved
+            torch.cuda.max_memory_cached = torch.xpu.max_memory_reserved
+            torch.cuda.reset_peak_memory_stats = torch.xpu.reset_peak_memory_stats
+            torch.cuda.reset_max_memory_cached = torch.xpu.reset_peak_memory_stats
+            torch.cuda.reset_max_memory_allocated = torch.xpu.reset_peak_memory_stats
+            torch.cuda.memory_stats_as_nested_dict = torch.xpu.memory_stats_as_nested_dict
+            torch.cuda.reset_accumulated_memory_stats = torch.xpu.reset_accumulated_memory_stats

            # RNG:
            torch.cuda.get_rng_state = torch.xpu.get_rng_state
@@ -185,7 +190,8 @@ def ipex_init(): # pylint: disable=too-many-statements
                torch._C._XpuDeviceProperties.minor = 1

            # Fix functions with ipex:
-            torch.cuda.mem_get_info = lambda device=None: [(torch.xpu.get_device_properties(device).total_memory - torch.xpu.memory_reserved(device)), torch.xpu.get_device_properties(device).total_memory]
+            torch.xpu.mem_get_info = lambda device=None: [(torch.xpu.get_device_properties(device).total_memory - torch.xpu.memory_reserved(device)), torch.xpu.get_device_properties(device).total_memory]
+            torch.cuda.mem_get_info = torch.xpu.mem_get_info
            torch._utils._get_available_device_type = lambda: "xpu"
            torch.has_cuda = True
            torch.cuda.has_half = True
@@ -194,19 +200,18 @@ def ipex_init(): # pylint: disable=too-many-statements
            torch.backends.cuda.is_built = lambda *args, **kwargs: True
            torch.version.cuda = "12.1"
            torch.cuda.get_arch_list = lambda: ["ats-m150", "pvc"]
-            torch.cuda.get_device_capability = lambda *args, **kwargs: [12,1]
+            torch.cuda.get_device_capability = lambda *args, **kwargs: (12,1)
            torch.cuda.get_device_properties.major = 12
            torch.cuda.get_device_properties.minor = 1
            torch.cuda.ipc_collect = lambda *args, **kwargs: None
            torch.cuda.utilization = lambda *args, **kwargs: 0

-            ipex_hijacks()
-            if not device_supports_fp64 or os.environ.get('IPEX_FORCE_ATTENTION_SLICE', None) is not None:
-                try:
-                    from .diffusers import ipex_diffusers
-                    ipex_diffusers()
-                except Exception: # pylint: disable=broad-exception-caught
-                    pass
+            ipex_hijacks(legacy=legacy)
+            try:
+                from .diffusers import ipex_diffusers
+                ipex_diffusers()
+            except Exception: # pylint: disable=broad-exception-caught
+                pass
            torch.cuda.is_xpu_hijacked = True
    except Exception as e:
        return False, e
@@ -1,16 +1,24 @@
 import os
+from functools import wraps, cache
 import torch
 import diffusers #0.29.1 # pylint: disable=import-error
 from diffusers.models.attention_processor import Attention
-from diffusers.models import transformers
-from diffusers.utils import USE_PEFT_BACKEND
-from functools import cache

 # pylint: disable=protected-access, missing-function-docstring, line-too-long

 device_supports_fp64 = torch.xpu.has_fp64_dtype() if hasattr(torch.xpu, "has_fp64_dtype") else torch.xpu.get_device_properties("xpu").has_fp64
 attention_slice_rate = float(os.environ.get('IPEX_ATTENTION_SLICE_RATE', 4))

+
+# Diffusers FreeU
+# Diffusers is imported before ipex hijacks so fourier_filter needs hijacking too
+original_fourier_filter = diffusers.utils.torch_utils.fourier_filter
+@wraps(diffusers.utils.torch_utils.fourier_filter)
+def fourier_filter(x_in, threshold, scale):
+    return_dtype = x_in.dtype
+    return original_fourier_filter(x_in.to(dtype=torch.float32), threshold, scale).to(dtype=return_dtype)
+
+
 # fp64 error
 def rope(pos: torch.Tensor, dim: int, theta: int) -> torch.Tensor:
    assert dim % 2 == 0, "The dimension must be even."
@@ -27,6 +35,7 @@ def rope(pos: torch.Tensor, dim: int, theta: int) -> torch.Tensor:
    out = stacked_out.view(batch_size, -1, dim // 2, 2, 2)
    return out.float()

+
@cache
 def find_slice_size(slice_size, slice_block_size):
    while (slice_size * slice_block_size) > attention_slice_rate:
@@ -74,6 +83,7 @@ def find_attention_slice_sizes(query_shape, query_element_size, query_device_typ

    return do_split, do_split_2, do_split_3, split_slice_size, split_2_slice_size, split_3_slice_size

+
 class SlicedAttnProcessor: # pylint: disable=too-few-public-methods
    r"""
    Processor for implementing sliced attention.
@@ -320,9 +330,11 @@ class AttnProcessor:

        return hidden_states

+
 def ipex_diffusers():
+    diffusers.utils.torch_utils.fourier_filter = fourier_filter
    #ARC GPUs can't allocate more than 4GB to a single block:
-    diffusers.models.attention_processor.SlicedAttnProcessor = SlicedAttnProcessor
-    diffusers.models.attention_processor.AttnProcessor = AttnProcessor
-    if not device_supports_fp64 and hasattr(transformers, "transformer_flux"):
+    if not device_supports_fp64 or os.environ.get('IPEX_FORCE_ATTENTION_SLICE', None) is not None:
+        diffusers.models.attention_processor.SlicedAttnProcessor = SlicedAttnProcessor
+        diffusers.models.attention_processor.AttnProcessor = AttnProcessor
        diffusers.models.transformers.transformer_flux.rope = rope
@@ -105,6 +105,20 @@ def scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.
        attn_mask = attn_mask.to(dtype=query.dtype)
    return original_scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, **kwargs)

+# Diffusers FreeU
+original_fft_fftn = torch.fft.fftn
+@wraps(torch.fft.fftn)
+def fft_fftn(input, s=None, dim=None, norm=None, *, out=None):
+    return_dtype = input.dtype
+    return original_fft_fftn(input.to(dtype=torch.float32), s=s, dim=dim, norm=norm, out=out).to(dtype=return_dtype)
+
+# Diffusers FreeU
+original_fft_ifftn = torch.fft.ifftn
+@wraps(torch.fft.ifftn)
+def fft_ifftn(input, s=None, dim=None, norm=None, *, out=None):
+    return_dtype = input.dtype
+    return original_fft_ifftn(input.to(dtype=torch.float32), s=s, dim=dim, norm=norm, out=out).to(dtype=return_dtype)
+
 # A1111 FP16
 original_functional_group_norm = torch.nn.functional.group_norm
@wraps(torch.nn.functional.group_norm)
@@ -220,7 +234,7 @@ def torch_empty(*args, device=None, **kwargs):
 original_torch_randn = torch.randn
@wraps(torch.randn)
 def torch_randn(*args, device=None, dtype=None, **kwargs):
-    if dtype == bytes:
+    if dtype is bytes:
        dtype = None
    if check_device(device):
        return original_torch_randn(*args, device=return_xpu(device), **kwargs)
@@ -279,7 +293,9 @@ def torch_load(f, map_location=None, *args, **kwargs):


 # Hijack Functions:
-def ipex_hijacks():
+def ipex_hijacks(legacy=True):
+    if legacy:
+        torch.nn.functional.interpolate = interpolate
    torch.tensor = torch_tensor
    torch.Tensor.to = Tensor_to
    torch.Tensor.cuda = Tensor_cuda
@@ -305,10 +321,11 @@ def ipex_hijacks():
    torch.nn.functional.layer_norm = functional_layer_norm
    torch.nn.functional.linear = functional_linear
    torch.nn.functional.conv2d = functional_conv2d
-    torch.nn.functional.interpolate = interpolate
    torch.nn.functional.pad = functional_pad

    torch.bmm = torch_bmm
+    torch.fft.fftn = fft_fftn
+    torch.fft.ifftn = fft_ifftn
    if not device_supports_fp64:
        torch.from_numpy = from_numpy
        torch.as_tensor = as_tensor
@@ -1,5 +1,6 @@
 import os
 import sys
+import time
 from collections import namedtuple
 from pathlib import Path
 import re
@@ -11,6 +12,24 @@ from torchvision.transforms.functional import InterpolationMode
 from modules import devices, paths, shared, lowvram, errors


+config = {
+    "caption_max_length": 64,
+    "chunk_size": 1024,
+    "flavor_intermediate_count": 1024,
+    "min_flavors": 2,
+    "max_flavors": 8,
+    "clip_offload": True,
+    "caption_offload": True,
+}
+caption_models = {
+    'blip-base': 'Salesforce/blip-image-captioning-base',
+    'blip-large': 'Salesforce/blip-image-captioning-large',
+    'blip2-opt-2.7b': 'Salesforce/blip2-opt-2.7b-coco',
+    'blip2-opt-6.7b': 'Salesforce/blip2-opt-6.7b',
+    'blip2-flip-t5-xl': 'Salesforce/blip2-flan-t5-xl',
+    'blip2-flip-t5-xxl': 'Salesforce/blip2-flan-t5-xxl',
+}
+ci = None
 blip_image_eval_size = 384
 clip_model_name = 'ViT-L/14'
 Category = namedtuple("Category", ["name", "topn", "items"])
@@ -48,7 +67,7 @@ class InterrogateModels:
        self.loaded_categories = None
        self.skip_categories = []
        self.content_dir = content_dir
-        self.running_on_cpu = devices.device_interrogate == torch.device("cpu")
+        self.running_on_cpu = False

    def categories(self):
        if not os.path.exists(self.content_dir):
@@ -104,7 +123,7 @@ class InterrogateModels:
        else:
            model, preprocess = clip.load(clip_model_name, download_root=shared.opts.clip_models_path)
        model.eval()
-        model = model.to(devices.device_interrogate)
+        model = model.to(devices.device)
        return model, preprocess

    def load(self):
@@ -112,12 +131,12 @@ class InterrogateModels:
            self.blip_model = self.load_blip_model()
            if not shared.opts.no_half and not self.running_on_cpu:
                self.blip_model = self.blip_model.half()
-        self.blip_model = self.blip_model.to(devices.device_interrogate)
+        self.blip_model = self.blip_model.to(devices.device)
        if self.clip_model is None:
            self.clip_model, self.clip_preprocess = self.load_clip_model()
            if not shared.opts.no_half and not self.running_on_cpu:
                self.clip_model = self.clip_model.half()
-        self.clip_model = self.clip_model.to(devices.device_interrogate)
+        self.clip_model = self.clip_model.to(devices.device)
        self.dtype = next(self.clip_model.parameters()).dtype

    def send_clip_to_ram(self):
@@ -141,10 +160,10 @@ class InterrogateModels:
        if shared.opts.interrogate_clip_dict_limit != 0:
            text_array = text_array[0:int(shared.opts.interrogate_clip_dict_limit)]
        top_count = min(top_count, len(text_array))
-        text_tokens = clip.tokenize(list(text_array), truncate=True).to(devices.device_interrogate)
+        text_tokens = clip.tokenize(list(text_array), truncate=True).to(devices.device)
        text_features = self.clip_model.encode_text(text_tokens).type(self.dtype)
        text_features /= text_features.norm(dim=-1, keepdim=True)
-        similarity = torch.zeros((1, len(text_array))).to(devices.device_interrogate)
+        similarity = torch.zeros((1, len(text_array))).to(devices.device)
        for i in range(image_features.shape[0]):
            similarity += (100.0 * image_features[i].unsqueeze(0) @ text_features.T).softmax(dim=-1)
        similarity /= image_features.shape[0]
@@ -156,7 +175,7 @@ class InterrogateModels:
            transforms.Resize((blip_image_eval_size, blip_image_eval_size), interpolation=InterpolationMode.BICUBIC),
            transforms.ToTensor(),
            transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))
-        ])(pil_image).unsqueeze(0).type(self.dtype).to(devices.device_interrogate)
+        ])(pil_image).unsqueeze(0).type(self.dtype).to(devices.device)
        with devices.inference_context():
            caption = self.blip_model.generate(gpu_image, sample=False, num_beams=shared.opts.interrogate_clip_num_beams, min_length=shared.opts.interrogate_clip_min_length, max_length=shared.opts.interrogate_clip_max_length)
        return caption[0]
@@ -180,7 +199,7 @@ class InterrogateModels:
            self.send_blip_to_ram()
            devices.torch_gc()
            res = caption
-            clip_image = self.clip_preprocess(pil_image).unsqueeze(0).type(self.dtype).to(devices.device_interrogate)
+            clip_image = self.clip_preprocess(pil_image).unsqueeze(0).type(self.dtype).to(devices.device)
            with devices.inference_context(), devices.autocast():
                image_features = self.clip_model.encode_image(clip_image).type(self.dtype)
                image_features /= image_features.norm(dim=-1, keepdim=True)
@@ -200,10 +219,6 @@ class InterrogateModels:

 # --------- interrrogate ui

-ci = None
-low_vram = False
-
-
 class BatchWriter:
    def __init__(self, folder):
        self.folder = folder
@@ -219,24 +234,56 @@ class BatchWriter:
            self.file.close()


+def update_interrogate_params(caption_max_length, chunk_size, min_flavors, max_flavors, flavor_intermediate_count):
+    config["caption_max_length"] = int(caption_max_length)
+    config["chunk_size"] = int(chunk_size)
+    config["min_flavors"] = int(min_flavors)
+    config["max_flavors"] = int(max_flavors)
+    config["flavor_intermediate_count"] = int(flavor_intermediate_count)
+    if ci is not None:
+        ci.config.caption_max_length = config["caption_max_length"]
+        ci.config.chunk_size = config["chunk_size"]
+        ci.config.flavor_intermediate_count = config["flavor_intermediate_count"]
+    shared.log.debug(f'Interrogate params: {config}')
+
 def get_clip_models():
    import open_clip
    return ['/'.join(x) for x in open_clip.list_pretrained()]


-def load_interrogator(model):
-    from clip_interrogator import Config, Interrogator
+def load_interrogator(clip_model, blip_model):
+    from installer import install
+    install('clip_interrogator==0.6.0')
+    import clip_interrogator
+    clip_interrogator.CAPTION_MODELS = caption_models
    global ci # pylint: disable=global-statement
    if ci is None:
-        config = Config(device=devices.get_optimal_device(), cache_path=os.path.join(paths.models_path, 'Interrogator'), clip_model_name=model, quiet=True)
-        if low_vram:
-            config.apply_low_vram_defaults()
-        shared.log.info(f'Interrogate load: config={config}')
-        ci = Interrogator(config)
-    elif model != ci.config.clip_model_name:
-        ci.config.clip_model_name = model
-        shared.log.info(f'Interrogate load: config={ci.config}')
+        interrogator_config = clip_interrogator.Config(
+            device=devices.get_optimal_device(),
+            cache_path=os.path.join(paths.models_path, 'Interrogator'),
+            clip_model_name=clip_model,
+            caption_model_name=blip_model,
+            quiet=True,
+            caption_max_length=config['caption_max_length'],
+            chunk_size=config['chunk_size'],
+            flavor_intermediate_count=config['flavor_intermediate_count'],
+            clip_offload=config['clip_offload'],
+            caption_offload=config['caption_offload'],
+        )
+        t0 = time.time()
+        ci = clip_interrogator.Interrogator(interrogator_config)
+        t1 = time.time()
+        shared.log.info(f'Interrogate load: config={ci.config} min_flavors={config["min_flavors"]} max_flavors={config["max_flavors"]} time={t1-t0:.2f}')
+    elif clip_model != ci.config.clip_model_name or blip_model != ci.config.caption_model_name:
+        t0 = time.time()
+        ci.config.clip_model_name = clip_model
+        ci.config.clip_model = None
        ci.load_clip_model()
+        ci.config.caption_model_name = blip_model
+        ci.config.caption_model = None
+        ci.load_caption_model()
+        t1 = time.time()
+        shared.log.info(f'Interrogate reload: config={ci.config} min_flavors={config["min_flavors"]} max_flavors={config["max_flavors"]} time={t1-t0:.2f}')


 def unload_clip_model():
@@ -250,32 +297,35 @@ def unload_clip_model():


 def interrogate(image, mode, caption=None):
-    shared.log.info(f'Interrogate: image={image} mode={mode} config={ci.config}')
+    shared.log.info(f'Interrogate: mode={mode} image={image}')
+    t0 = time.time()
    if mode == 'best':
-        prompt = ci.interrogate(image, caption=caption)
+        prompt = ci.interrogate(image, caption=caption, min_flavors=config["min_flavors"], max_flavors=config["max_flavors"])
    elif mode == 'caption':
        prompt = ci.generate_caption(image) if caption is None else caption
    elif mode == 'classic':
-        prompt = ci.interrogate_classic(image, caption=caption)
+        prompt = ci.interrogate_classic(image, caption=caption, max_flavors=config["max_flavors"])
    elif mode == 'fast':
-        prompt = ci.interrogate_fast(image, caption=caption)
+        prompt = ci.interrogate_fast(image, caption=caption, max_flavors=config["max_flavors"])
    elif mode == 'negative':
-        prompt = ci.interrogate_negative(image)
+        prompt = ci.interrogate_negative(image, max_flavors=config["max_flavors"])
    else:
        raise RuntimeError(f"Unknown mode {mode}")
+    t1 = time.time()
+    shared.log.debug(f'Interrogate: prompt="{prompt}" time={t1-t0:.2f}')
    return prompt


-def interrogate_image(image, model, mode):
+def interrogate_image(image, clip_model, blip_model, mode):
    shared.state.begin('Interrogate')
    try:
        if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram):
            lowvram.send_everything_to_cpu()
            devices.torch_gc()
-        load_interrogator(model)
+        load_interrogator(clip_model, blip_model)
        image = image.convert('RGB')
-        shared.log.info(f'Interrogate: image={image} mode={mode} config={ci.config}')
        prompt = interrogate(image, mode)
+        devices.torch_gc()
    except Exception as e:
        prompt = f"Exception {type(e)}"
        shared.log.error(f'Interrogate: {e}')
@@ -283,7 +333,7 @@ def interrogate_image(image, model, mode):
    return prompt


-def interrogate_batch(batch_files, batch_folder, batch_str, model, mode, write):
+def interrogate_batch(batch_files, batch_folder, batch_str, clip_model, blip_model, mode, write):
    files = []
    if batch_files is not None:
        files += [f.name for f in batch_files]
@@ -300,7 +350,7 @@ def interrogate_batch(batch_files, batch_folder, batch_str, model, mode, write):
        if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram):
            lowvram.send_everything_to_cpu()
            devices.torch_gc()
-        load_interrogator(model)
+        load_interrogator(clip_model, blip_model)
        shared.log.info(f'Interrogate batch: images={len(files)} mode={mode} config={ci.config}')
        captions = []
        # first pass: generate captions
@@ -339,8 +389,8 @@ def interrogate_batch(batch_files, batch_folder, batch_str, model, mode, write):
    return '\n\n'.join(prompts)


-def analyze_image(image, model):
-    load_interrogator(model)
+def analyze_image(image, clip_model, blip_model):
+    load_interrogator(clip_model, blip_model)
    image = image.convert('RGB')
    image_features = ci.image_to_features(image)
    top_mediums = ci.mediums.rank(image_features, 5)
@@ -83,7 +83,7 @@ def crop_images(images, crops):
    try:
        for i in range(len(images)):
            if crops[i]:
-                from scripts.face_details import yolo # pylint: disable=no-name-in-module
+                from shared import yolo # pylint: disable=no-name-in-module
                yolo.load()
                cropped = []
                for image in images[i]:
@@ -240,5 +240,5 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt
        t1 = time.time()
        shared.log.info(f'IP adapter: {ip_str} image={adapter_images} mask={adapter_masks is not None} time={t1-t0:.2f}')
    except Exception as e:
-        shared.log.error(f'IP adapter failed to load: repo={base_repo} folder={ip_subfolder} weights={adapters} names={adapter_names} {e}')
+        shared.log.error(f'IP adapter failed to load: repo="{base_repo}" folder="{ip_subfolder}" weights={adapters} names={adapter_names} {e}')
    return True
@@ -72,7 +72,7 @@ def download_model():
    filename = os.path.basename(parts.path)
    cached_file = os.path.join(model_dir, filename)
    if not os.path.exists(cached_file):
-        log.info(f'LaMa download: url={LAMA_MODEL_URL} file={cached_file}')
+        log.info(f'LaMa download: url="{LAMA_MODEL_URL}" file="{cached_file}"')
        hash_prefix = None
        download_url_to_file(LAMA_MODEL_URL, cached_file, hash_prefix, progress=True)
    return cached_file
@@ -0,0 +1,44 @@
+from modules import shared, sd_models, devices
+from .linfusion import LinFusion
+from .attention import GeneralizedLinearAttention
+
+
+applied: LinFusion = None
+
+
+def detect(pipeline):
+    if pipeline.__class__.__name__ == 'StableDiffusionXLPipeline':
+        return "Yuanshi/LinFusion-XL"
+    if pipeline.__class__.__name__ == 'StableDiffusionPipeline':
+        return "Yuanshi/LinFusion-1-5"
+    return None
+
+
+def apply(pipeline, pretrained: bool = True):
+    global applied # pylint: disable=global-statement
+    if applied is not None:
+        return
+    # linfusion = LinFusion.construct_for(pipeline=pipeline)
+    if not pretrained:
+        model_path = None
+        default_config = LinFusion.get_default_config(unet=pipeline.unet)
+        applied = LinFusion(**default_config).to(device=pipeline.unet.device, dtype=pipeline.unet.dtype)
+        applied.mount_to(unet=pipeline.unet)
+    else:
+        model_path = detect(pipeline)
+        if model_path is None:
+            shared.log.error('LinFusion: unsupported model type')
+            return
+        applied = LinFusion.from_pretrained(model_path, cache_dir=shared.opts.hfcache_dir).to(device=pipeline.unet.device, dtype=pipeline.unet.dtype)
+        applied.mount_to(unet=pipeline.unet)
+    shared.log.debug(f'LinFusion: apply class={applied.__class__.__name__} model="{model_path}" modules={len(applied.modules_dict)}')
+
+
+def unapply(pipeline):
+    global applied # pylint: disable=global-statement
+    if applied is None:
+        return
+    shared.log.debug('LinFusion: unapply')
+    sd_models.set_diffusers_attention(pipeline)
+    devices.torch_gc()
+    applied = None
--- a/Show More
+++ b/Show More