Merge branch 'dev' into Dev/DirectoryCacheing

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log for SD.Next
 
-## Update for 2023-01-05
+## Update for 2023-01-06
 
 Following-up on a major release, some more functionality in new Control module  
 And it also includes fixes for all reported issues so far  
@@ -21,6 +21,8 @@ And it also includes fixes for all reported issues so far
   - add **openpose xl** controlnet  
   - configurable output folder in settings  
   - auto-refresh available models on tab activate  
+  - add image preview for override images set per-unit  
+  - more compact unit layout  
   - reduce usage of temp files  
   - add context menu to action buttons  
   - resize by now applies to input image or frame individually  
@@ -30,10 +32,21 @@ And it also includes fixes for all reported issues so far
   - fix correct image mode  
   - fix batch/folder/video modes  
   - fix pipeline switching between different modes  
+- [FaceID](https://huggingface.co/h94/IP-Adapter-FaceID)  
+  full implementation for *SD15* and *SD-XL*, to use simply select from *Scripts*  
+  - **Base** (93MB) uses *InsightFace* to generate face embeds and *OpenCLIP-ViT-H-14* (2.5GB) as image encoder  
+  - **SXDL** (1022MB) uses *InsightFace* to generate face embeds and *OpenCLIP-ViT-bigG-14* (3.7GB) as image encoder  
+  - **Plus** (150MB) uses *InsightFace* to generate face embeds and *CLIP-ViT-H-14-laion2B* (3.8GB) as image encoder  
+  *note*: all models are downloaded on first use  
+- [IPAdapter](https://huggingface.co/h94/IP-Adapter)  
+  additional models for *SD15* and *SD-XL*, to use simply select from *Scripts*:  
+  - **SD15**: Base, Base ViT-G, Light, Plus, Plus Face, Full Face  
+  - **SDXL**: Base SXDL, Base ViT-H SXDL, Plus ViT-H SXDL, Plus Face ViT-H SXDL  
 - **Improvements**  
   - **server startup**: performance  
-    - faster extension load
-    - faster json parsing
+    - faster extension load  
+    - faster json parsing  
+    - faster lora indexing  
   - **offline deployment**: allow deployment without git clone  
     for example, you can now deploy a zip of the sdnext folder  
   - **latent upscale**: updated latent upscalers (some are new)  
@@ -44,7 +57,7 @@ And it also includes fixes for all reported issues so far
     - enable vae tiling  
     - add autodetect optimial value  
       set tile size to 0 to use autodetected value  
-  - **cli**: 
+  - **cli**  
     - `sdapi.py` allow manual api invoke  
       example: `python cli/sdapi.py /sdapi/v1/sd-models`  
     - `image-exif.py` improve metadata parsing  
@@ -67,6 +80,7 @@ And it also includes fixes for all reported issues so far
     requires nightly versions of `torch` and `torchao`  
     > pip install -U --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121  
     > pip install -U git+https://github.com/pytorch-labs/ao  
+  - new option: **compile text encoder** (experimental)  
 - **IPEX**, thanks @disty0  
   - rewrite ipex hijacks without CondFunc  
     improves compatibilty and performance  
@@ -77,6 +91,8 @@ And it also includes fixes for all reported issues so far
   - **4-bit support with NNCF**  
     enable *Compress Model weights with NNCF* from *Compute Settings* and set a 4-bit NNCF mode  
     4-bit and 8-bit with OpenVINO is CPU only for now  
+  - experimental support for *Text Encoder* compiling  
+    OpenVINO is faster than IPEX now  
   - reduce system memory usage after compile  
   - fix cache loading with multiple models  
 - **Fixes**  
@@ -91,6 +107,7 @@ And it also includes fixes for all reported issues so far
   - processing: correct display metadata  
   - live preview: fix when using `bfloat16`
   - upscale: fix ldsr
+  - cli: fix cmd args parsing  
 
 ## Update for 2023-12-29
 

extensions-builtin/Lora/network.py

@@ -24,20 +24,8 @@ class NetworkOnDisk:
         self.metadata = {}
         self.is_safetensors = os.path.splitext(filename)[1].lower() == ".safetensors"
 
-        def read_metadata(): #  # pylint: disable=W0612
-            metadata = sd_models.read_metadata_from_safetensors(filename)
-            metadata.pop('ssmd_cover_images', None)  # those are cover images, and they are too big to display in UI as text
-            return metadata
-
         if self.is_safetensors:
             self.metadata = sd_models.read_metadata_from_safetensors(filename)
-            """
-            try:
-                self.metadata = cache.cached_data_for_file('safetensors-metadata', "lora/" + self.name, filename, read_metadata)
-            except Exception as e:
-                errors.display(e, f"reading lora {filename}")
-            """
-
         if self.metadata:
             m = {}
             for k, v in sorted(self.metadata.items(), key=lambda x: metadata_tags_order.get(x[0], 999)):
@@ -46,11 +34,7 @@ class NetworkOnDisk:
         self.alias = self.metadata.get('ss_output_name', self.name)
         self.hash = None
         self.shorthash = None
-        self.set_hash(
-            self.metadata.get('sshs_model_hash') or
-            hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or
-            ''
-        )
+        self.set_hash(self.metadata.get('sshs_model_hash') or hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or '')
         self.sd_version = self.detect_version()
 
     def detect_version(self):
@@ -65,9 +49,6 @@ class NetworkOnDisk:
     def set_hash(self, v):
         self.hash = v
         self.shorthash = self.hash[0:12]
-        if self.shorthash:
-            import networks
-            networks.available_network_hash_lookup[self.shorthash] = self
 
     def read_hash(self):
         if not self.hash:

extensions-builtin/Lora/networks.py

@@ -2,7 +2,7 @@ from typing import Union, List
 import os
 import re
 import time
-from threading import Thread
+import concurrent
 import lora_patches
 import network
 import network_lora
@@ -441,18 +441,25 @@ def list_available_networks():
         shared.log.warning('LoRA directory not found: path="{shared.cmd_opts.lora_dir}"')
     if os.path.exists(shared.cmd_opts.lyco_dir):
         directories.append(shared.cmd_opts.lyco_dir)
-    for filename in files_cache.list_files(*directories, ext_filter=[".pt", ".ckpt", ".safetensors"]):
+    def add_network(filename):
+        if os.path.isdir(filename):
+            return
         name = os.path.splitext(os.path.basename(filename))[0]
         try:
             entry = network.NetworkOnDisk(name, filename)
+            available_networks[entry.name] = entry
+            if entry.alias in available_network_aliases:
+                forbidden_network_aliases[entry.alias.lower()] = 1
+            available_network_aliases[entry.name] = entry
+            available_network_aliases[entry.alias] = entry
+            if entry.shorthash:
+                available_network_hash_lookup[entry.shorthash] = entry
         except OSError as e:  # should catch FileNotFoundError and PermissionError etc.
             shared.log.error(f"Failed to load network {name} from {filename} {e}")
-            continue
-        available_networks[name] = entry
-        if entry.alias in available_network_aliases:
-            forbidden_network_aliases[entry.alias.lower()] = 1
-        available_network_aliases[name] = entry
-        available_network_aliases[entry.alias] = entry
+            
+    with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor:
+        for fn in files_cache.list_files(*directories, ext_filter=[".pt", ".ckpt", ".safetensors"]):
+            executor.submit(add_network, fn)
     print(f'Lora/LyCORIS Networks: networks={len(available_networks)} directories={directories}')
 
 
@@ -478,5 +485,4 @@ def infotext_pasted(infotext, params): # pylint: disable=W0613
         params["Prompt"] += "\n" + "".join(added)
 
 
-thread_lora = Thread(target=list_available_networks)
-thread_lora.start()
+list_available_networks()

javascript/sdnext.css

@@ -261,7 +261,9 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
 #control_input_type { max-width: 18em }
 #control_settings .small-accordion .form { min-width: 350px !important }
 .control-button { min-height: 42px; max-height: 42px; line-height: 1em; }
-.control-tabs>.tab-nav { margin-bottom: 0; margin-top: 0; }
+.control-tabs > .tab-nav { margin-bottom: 0; margin-top: 0; }
+.control-unit { max-width: 1200px; padding: 0 !important; margin-top: -10px !important; }
+.control-unit > .label-wrap { margin-bottom: 0 !important; }
 .processor-settings { padding: 0 !important; max-width: 300px; }
 .processor-group>div { flex-flow: wrap;gap: 1em; }
 

modules/cmd_args.py

@@ -80,12 +80,12 @@ def compatibility_args(opts, args):
     group.add_argument("--swinir-models-path", help=argparse.SUPPRESS, default=opts.swinir_models_path)
     group.add_argument("--ldsr-models-path", help=argparse.SUPPRESS, default=opts.ldsr_models_path)
     group.add_argument("--clip-models-path", type=str, help=argparse.SUPPRESS, default=opts.clip_models_path)
-    group.add_argument("--opt-channelslast", help=argparse.SUPPRESS, default=opts.opt_channelslast)
+    group.add_argument("--opt-channelslast", help=argparse.SUPPRESS, action='store_true', default=opts.opt_channelslast)
     group.add_argument("--xformers", default=(opts.cross_attention_optimization == "xFormers"), action='store_true', help=argparse.SUPPRESS)
-    group.add_argument("--disable-nan-check", help=argparse.SUPPRESS, default=opts.disable_nan_check)
+    group.add_argument("--disable-nan-check", help=argparse.SUPPRESS, action='store_true', default=opts.disable_nan_check)
     group.add_argument("--rollback-vae", help=argparse.SUPPRESS, default=opts.rollback_vae)
-    group.add_argument("--no-half", help=argparse.SUPPRESS, default=opts.no_half)
-    group.add_argument("--no-half-vae", help=argparse.SUPPRESS, default=opts.no_half_vae)
+    group.add_argument("--no-half", help=argparse.SUPPRESS, action='store_true', default=opts.no_half)
+    group.add_argument("--no-half-vae", help=argparse.SUPPRESS, action='store_true', default=opts.no_half_vae)
     group.add_argument("--precision", help=argparse.SUPPRESS, default=opts.precision)
     group.add_argument("--sub-quad-q-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_q_chunk_size)
     group.add_argument("--sub-quad-kv-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_kv_chunk_size)
@@ -94,7 +94,7 @@ def compatibility_args(opts, args):
     group.add_argument("--lyco-dir", help=argparse.SUPPRESS, default=opts.lyco_dir)
     group.add_argument("--embeddings-dir", help=argparse.SUPPRESS, default=opts.embeddings_dir)
     group.add_argument("--hypernetwork-dir", help=argparse.SUPPRESS, default=opts.hypernetwork_dir)
-    group.add_argument("--lyco-patch-lora", help=argparse.SUPPRESS, default=False)
+    group.add_argument("--lyco-patch-lora", help=argparse.SUPPRESS, action='store_true', default=False)
     group.add_argument("--lyco-debug", help=argparse.SUPPRESS, action='store_true', default=False)
     group.add_argument("--enable-console-prompts", help=argparse.SUPPRESS, action='store_true', default=False)
     group.add_argument("--safe", help=argparse.SUPPRESS, action='store_true', default=False)

modules/control/run.py

@@ -467,11 +467,9 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
                     # pipeline
                     output = None
                     if pipe is not None: # run new pipeline
-                        debug(f'Control exec pipeline: class={pipe.__class__}')
-                        debug(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)}')
+                        debug(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)} class={pipe.__class__}')
                         debug(f'Control exec pipeline: p={vars(p)}')
-                        debug(f'Control exec pipeline: args={p.task_args}')
-                        debug(f'Control exec pipeline: image={p.task_args.get("image", None)} control={p.task_args.get("control_image", None)} mask={p.task_args.get("mask_image", None)} ref={p.task_args.get("ref_image", None)}')
+                        debug(f'Control exec pipeline: args={p.task_args} image={p.task_args.get("image", None)} control={p.task_args.get("control_image", None)} mask={p.task_args.get("mask_image", None)} ref={p.task_args.get("ref_image", None)}')
                         processed: processing.Processed = processing.process_images(p) # run actual pipeline
                         output = processed.images if processed is not None else None
                         # output = pipe(**vars(p)).images # alternative direct pipe exec call

modules/control/unit.py

@@ -1,5 +1,6 @@
 from typing import Union
 from PIL import Image
+import gradio as gr
 from modules.shared import log
 from modules.control import processors
 from modules.control.units import controlnet
@@ -31,6 +32,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
                  image_input = None,
                  preview_process = None,
                  image_upload = None,
+                 image_preview = None,
                  control_start = None,
                  control_end = None,
                  result_txt = None,
@@ -101,8 +103,10 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
                 self.process.override = Image.open(image_file.name)
                 self.override = self.process.override
                 log.debug(f'Control process upload image: path="{image_file.name}" image={self.process.override}')
+                return gr.update(visible=self.process.override is not None, value=self.process.override)
             except Exception as e:
                 log.error(f'Control process upload image failed: path="{image_file.name}" error={e}')
+                return gr.update(visible=False, value=None)
 
         # actual init
         if self.type == 'adapter':
@@ -157,7 +161,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
         if preview_btn is not None:
             preview_btn.click(fn=self.process.preview, inputs=[self.input], outputs=[preview_process]) # return list of images for gallery
         if image_upload is not None:
-            image_upload.upload(fn=upload_image, inputs=[image_upload], outputs=[]) # return list of images for gallery
+            image_upload.upload(fn=upload_image, inputs=[image_upload], outputs=[image_preview]) # return list of images for gallery
         if control_start is not None and control_end is not None:
             control_start.change(fn=control_change, inputs=[control_start, control_end])
             control_end.change(fn=control_change, inputs=[control_start, control_end])

modules/control/units/controlnet.py

@@ -3,7 +3,7 @@ import time
 from typing import Union
 from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, ControlNetModel, StableDiffusionControlNetPipeline, StableDiffusionXLControlNetPipeline
 from modules.control.units import detect
-from modules.shared import log, opts
+from modules.shared import log, opts, listdir
 from modules import errors
 
 
@@ -44,7 +44,7 @@ cache_dir = 'models/control/controlnet'
 
 def find_models():
     path = os.path.join(opts.control_dir, 'controlnet')
-    files = os.listdir(path)
+    files = listdir(path)
     files = [f for f in files if f.endswith('.safetensors')]
     downloaded_models = {}
     for f in files:

modules/control/units/lite.py

@@ -4,7 +4,7 @@ from typing import Union
 import numpy as np
 from PIL import Image
 from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline
-from modules.shared import log, opts
+from modules.shared import log, opts, listdir
 from modules import errors
 from modules.control.units.lite_model import ControlNetLLLite
 
@@ -31,7 +31,7 @@ cache_dir = 'models/control/lite'
 
 def find_models():
     path = os.path.join(opts.control_dir, 'lite')
-    files = os.listdir(path)
+    files = listdir(path)
     files = [f for f in files if f.endswith('.safetensors')]
     downloaded_models = {}
     for f in files:

modules/control/units/reference.py

@@ -3,7 +3,7 @@ import time
 from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline
 from modules.control.proc.reference_sd15 import StableDiffusionReferencePipeline
 from modules.control.proc.reference_sdxl import StableDiffusionXLReferencePipeline
-from modules.shared import log
+from modules.shared import log, opts
 from modules.control.units import detect
 
 
@@ -22,6 +22,8 @@ class ReferencePipeline():
         if pipeline is None:
             log.error(f'Control {what} model pipeline: model not loaded')
             return
+        if opts.diffusers_fuse_projections and hasattr(pipeline, 'unfuse_qkv_projections'):
+            pipeline.unfuse_qkv_projections()
         if detect.is_sdxl(pipeline):
             self.pipeline = StableDiffusionXLReferencePipeline(
                 vae=pipeline.vae,

modules/control/units/xs.py

@@ -2,7 +2,7 @@ import os
 import time
 from typing import Union
 from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline
-from modules.shared import log, opts
+from modules.shared import log, opts, listdir
 from modules import errors
 from modules.control.units.xs_model import ControlNetXSModel
 from modules.control.units.xs_pipe import StableDiffusionControlNetXSPipeline, StableDiffusionXLControlNetXSPipeline
@@ -27,7 +27,7 @@ cache_dir = 'models/control/xs'
 
 def find_models():
     path = os.path.join(opts.control_dir, 'xs')
-    files = os.listdir(path)
+    files = listdir(path)
     files = [f for f in files if f.endswith('.safetensors')]
     downloaded_models = {}
     for f in files:

modules/images.py

@@ -504,7 +504,7 @@ def get_next_sequence_number(path, basename):
     prefix_length = len(basename)
     if not os.path.isdir(path):
         return 0
-    for p in os.listdir(path):
+    for p in shared.listdir(path):
         if p.startswith(basename):
             parts = os.path.splitext(p[prefix_length:])[0].split('-')  # splits the filename (removing the basename first if one is defined, so the sequence number is always the first element)
             try:

modules/img2img.py

@@ -22,10 +22,10 @@ def process_batch(p, input_files, input_dir, output_dir, inpaint_mask_dir, args)
         if not os.path.isdir(input_dir):
             shared.log.error(f"Process batch: directory not found: {input_dir}")
             return
-        image_files = shared.listfiles(input_dir)
+        image_files = shared.listdir(input_dir)
     is_inpaint_batch = False
     if inpaint_mask_dir:
-        inpaint_masks = shared.listfiles(inpaint_mask_dir)
+        inpaint_masks = shared.listdir(inpaint_mask_dir)
         is_inpaint_batch = len(inpaint_masks) > 0
     if is_inpaint_batch:
         shared.log.info(f"Process batch: inpaint batch masks={len(inpaint_masks)}")

modules/intel/ipex/hijacks.py

@@ -1,4 +1,5 @@
-import contextlib
+from functools import wraps
+from contextlib import nullcontext
 import torch
 import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
 from modules import devices
@@ -12,7 +13,7 @@ class DummyDataParallel(torch.nn.Module): # pylint: disable=missing-class-docstr
         return module.to(devices.device)
 
 def return_null_context(*args, **kwargs): # pylint: disable=unused-argument
-    return contextlib.nullcontext()
+    return nullcontext()
 
 @property
 def is_cuda(self):
@@ -27,6 +28,7 @@ def return_xpu(device):
 
 # Autocast
 original_autocast = torch.autocast
+@wraps(torch.autocast)
 def ipex_autocast(*args, **kwargs):
     if len(args) > 0 and (args[0] == "cuda" or args[0] == "xpu"):
         if "dtype" in kwargs:
@@ -38,6 +40,7 @@ def ipex_autocast(*args, **kwargs):
 
 # Latent Antialias CPU Offload:
 original_interpolate = torch.nn.functional.interpolate
+@wraps(torch.nn.functional.interpolate)
 def interpolate(tensor, size=None, scale_factor=None, mode='nearest', align_corners=None, recompute_scale_factor=None, antialias=False): # pylint: disable=too-many-arguments
     if antialias or align_corners is not None:
         return_device = tensor.device
@@ -50,6 +53,7 @@ def interpolate(tensor, size=None, scale_factor=None, mode='nearest', align_corn
 
 # Diffusers Float64 (Alchemist GPUs doesn't support 64 bit):
 original_from_numpy = torch.from_numpy
+@wraps(torch.from_numpy)
 def from_numpy(ndarray):
     if ndarray.dtype == float:
         return original_from_numpy(ndarray.astype('float32'))
@@ -70,11 +74,13 @@ else:
 
 
 # Data Type Errors:
+@wraps(torch.bmm)
 def torch_bmm(input, mat2, *, out=None):
     if input.dtype != mat2.dtype:
         mat2 = mat2.to(input.dtype)
     return original_torch_bmm(input, mat2, out=out)
 
+@wraps(torch.nn.functional.scaled_dot_product_attention)
 def scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False):
     if query.dtype != key.dtype:
         key = key.to(dtype=query.dtype)
@@ -84,6 +90,7 @@ def scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.
 
 # A1111 FP16
 original_functional_group_norm = torch.nn.functional.group_norm
+@wraps(torch.nn.functional.group_norm)
 def functional_group_norm(input, num_groups, weight=None, bias=None, eps=1e-05):
     if weight is not None and input.dtype != weight.data.dtype:
         input = input.to(dtype=weight.data.dtype)
@@ -93,6 +100,7 @@ def functional_group_norm(input, num_groups, weight=None, bias=None, eps=1e-05):
 
 # A1111 BF16
 original_functional_layer_norm = torch.nn.functional.layer_norm
+@wraps(torch.nn.functional.layer_norm)
 def functional_layer_norm(input, normalized_shape, weight=None, bias=None, eps=1e-05):
     if weight is not None and input.dtype != weight.data.dtype:
         input = input.to(dtype=weight.data.dtype)
@@ -102,6 +110,7 @@ def functional_layer_norm(input, normalized_shape, weight=None, bias=None, eps=1
 
 # Training
 original_functional_linear = torch.nn.functional.linear
+@wraps(torch.nn.functional.linear)
 def functional_linear(input, weight, bias=None):
     if input.dtype != weight.data.dtype:
         input = input.to(dtype=weight.data.dtype)
@@ -110,6 +119,7 @@ def functional_linear(input, weight, bias=None):
     return original_functional_linear(input, weight, bias=bias)
 
 original_functional_conv2d = torch.nn.functional.conv2d
+@wraps(torch.nn.functional.conv2d)
 def functional_conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1):
     if input.dtype != weight.data.dtype:
         input = input.to(dtype=weight.data.dtype)
@@ -119,6 +129,7 @@ def functional_conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1,
 
 # A1111 Embedding BF16
 original_torch_cat = torch.cat
+@wraps(torch.cat)
 def torch_cat(tensor, *args, **kwargs):
     if len(tensor) == 3 and (tensor[0].dtype != tensor[1].dtype or tensor[2].dtype != tensor[1].dtype):
         return original_torch_cat([tensor[0].to(tensor[1].dtype), tensor[1], tensor[2].to(tensor[1].dtype)], *args, **kwargs)
@@ -127,6 +138,7 @@ def torch_cat(tensor, *args, **kwargs):
 
 # SwinIR BF16:
 original_functional_pad = torch.nn.functional.pad
+@wraps(torch.nn.functional.pad)
 def functional_pad(input, pad, mode='constant', value=None):
     if mode == 'reflect' and input.dtype == torch.bfloat16:
         return original_functional_pad(input.to(torch.float32), pad, mode=mode, value=value).to(dtype=torch.bfloat16)
@@ -135,6 +147,7 @@ def functional_pad(input, pad, mode='constant', value=None):
 
 
 original_torch_tensor = torch.tensor
+@wraps(torch.tensor)
 def torch_tensor(*args, device=None, **kwargs):
     if check_device(device):
         return original_torch_tensor(*args, device=return_xpu(device), **kwargs)
@@ -142,6 +155,7 @@ def torch_tensor(*args, device=None, **kwargs):
         return original_torch_tensor(*args, device=device, **kwargs)
 
 original_Tensor_to = torch.Tensor.to
+@wraps(torch.Tensor.to)
 def Tensor_to(self, device=None, *args, **kwargs):
     if check_device(device):
         return original_Tensor_to(self, return_xpu(device), *args, **kwargs)
@@ -149,6 +163,7 @@ def Tensor_to(self, device=None, *args, **kwargs):
         return original_Tensor_to(self, device, *args, **kwargs)
 
 original_Tensor_cuda = torch.Tensor.cuda
+@wraps(torch.Tensor.cuda)
 def Tensor_cuda(self, device=None, *args, **kwargs):
     if check_device(device):
         return original_Tensor_cuda(self, return_xpu(device), *args, **kwargs)
@@ -156,6 +171,7 @@ def Tensor_cuda(self, device=None, *args, **kwargs):
         return original_Tensor_cuda(self, device, *args, **kwargs)
 
 original_UntypedStorage_init = torch.UntypedStorage.__init__
+@wraps(torch.UntypedStorage.__init__)
 def UntypedStorage_init(*args, device=None, **kwargs):
     if check_device(device):
         return original_UntypedStorage_init(*args, device=return_xpu(device), **kwargs)
@@ -163,6 +179,7 @@ def UntypedStorage_init(*args, device=None, **kwargs):
         return original_UntypedStorage_init(*args, device=device, **kwargs)
 
 original_UntypedStorage_cuda = torch.UntypedStorage.cuda
+@wraps(torch.UntypedStorage.cuda)
 def UntypedStorage_cuda(self, device=None, *args, **kwargs):
     if check_device(device):
         return original_UntypedStorage_cuda(self, return_xpu(device), *args, **kwargs)
@@ -170,6 +187,7 @@ def UntypedStorage_cuda(self, device=None, *args, **kwargs):
         return original_UntypedStorage_cuda(self, device, *args, **kwargs)
 
 original_torch_empty = torch.empty
+@wraps(torch.empty)
 def torch_empty(*args, device=None, **kwargs):
     if check_device(device):
         return original_torch_empty(*args, device=return_xpu(device), **kwargs)
@@ -177,6 +195,7 @@ def torch_empty(*args, device=None, **kwargs):
         return original_torch_empty(*args, device=device, **kwargs)
 
 original_torch_randn = torch.randn
+@wraps(torch.randn)
 def torch_randn(*args, device=None, **kwargs):
     if check_device(device):
         return original_torch_randn(*args, device=return_xpu(device), **kwargs)
@@ -184,6 +203,7 @@ def torch_randn(*args, device=None, **kwargs):
         return original_torch_randn(*args, device=device, **kwargs)
 
 original_torch_ones = torch.ones
+@wraps(torch.ones)
 def torch_ones(*args, device=None, **kwargs):
     if check_device(device):
         return original_torch_ones(*args, device=return_xpu(device), **kwargs)
@@ -191,6 +211,7 @@ def torch_ones(*args, device=None, **kwargs):
         return original_torch_ones(*args, device=device, **kwargs)
 
 original_torch_zeros = torch.zeros
+@wraps(torch.zeros)
 def torch_zeros(*args, device=None, **kwargs):
     if check_device(device):
         return original_torch_zeros(*args, device=return_xpu(device), **kwargs)
@@ -198,6 +219,7 @@ def torch_zeros(*args, device=None, **kwargs):
         return original_torch_zeros(*args, device=device, **kwargs)
 
 original_torch_linspace = torch.linspace
+@wraps(torch.linspace)
 def torch_linspace(*args, device=None, **kwargs):
     if check_device(device):
         return original_torch_linspace(*args, device=return_xpu(device), **kwargs)
@@ -205,6 +227,7 @@ def torch_linspace(*args, device=None, **kwargs):
         return original_torch_linspace(*args, device=device, **kwargs)
 
 original_torch_Generator = torch.Generator
+@wraps(torch.Generator)
 def torch_Generator(device=None):
     if check_device(device):
         return original_torch_Generator(return_xpu(device))
@@ -212,6 +235,7 @@ def torch_Generator(device=None):
         return original_torch_Generator(device)
 
 original_torch_load = torch.load
+@wraps(torch.load)
 def torch_load(f, map_location=None, pickle_module=None, *, weights_only=False, mmap=None, **kwargs):
     if check_device(map_location):
         return original_torch_load(f, map_location=return_xpu(map_location), pickle_module=pickle_module, weights_only=weights_only, mmap=mmap, **kwargs)

modules/intel/openvino/__init__.py

@@ -229,8 +229,8 @@ def openvino_compile(gm: GraphModule, *args, model_hash_str: str = None, file_na
         om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype])
         om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape)))
     om.validate_nodes_and_infer_types()
-    if shared.opts.nncf_compress_weights and not (shared.compiled_model_state.compiling_vae and not shared.opts.nncf_compress_vae_weights):
-        if shared.compiled_model_state.compiling_vae or shared.opts.nncf_compress_weights_mode == "INT8":
+    if shared.opts.nncf_compress_weights and not (shared.compiled_model_state.compile_dont_use_4bit and not shared.opts.nncf_compress_vae_weights):
+        if shared.compiled_model_state.compile_dont_use_4bit or shared.opts.nncf_compress_weights_mode == "INT8":
             om = nncf.compress_weights(om)
         else:
             om = nncf.compress_weights(om, mode=getattr(nncf.CompressWeightsMode, shared.opts.nncf_compress_weights_mode), group_size=8, ratio=shared.opts.nncf_compress_weights_raito)
@@ -238,7 +238,7 @@ def openvino_compile(gm: GraphModule, *args, model_hash_str: str = None, file_na
     if model_hash_str is not None:
         core.set_property({'CACHE_DIR': cache_root + '/blob'})
 
-    shared.compiled_model_state.compiling_vae = False
+    shared.compiled_model_state.compile_dont_use_4bit = False
     compiled_model = core.compile_model(om, device)
     return compiled_model
 
@@ -261,15 +261,15 @@ def openvino_compile_cached_model(cached_model_path, *example_inputs):
         om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype])
         om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape)))
     om.validate_nodes_and_infer_types()
-    if shared.opts.nncf_compress_weights and not (shared.compiled_model_state.compiling_vae and not shared.opts.nncf_compress_vae_weights):
-        if shared.compiled_model_state.compiling_vae or shared.opts.nncf_compress_weights_mode == "INT8":
+    if shared.opts.nncf_compress_weights and not (shared.compiled_model_state.compile_dont_use_4bit and not shared.opts.nncf_compress_vae_weights):
+        if shared.compiled_model_state.compile_dont_use_4bit or shared.opts.nncf_compress_weights_mode == "INT8":
             om = nncf.compress_weights(om)
         else:
             om = nncf.compress_weights(om, mode=getattr(nncf.CompressWeightsMode, shared.opts.nncf_compress_weights_mode), group_size=8, ratio=shared.opts.nncf_compress_weights_raito)
 
     core.set_property({'CACHE_DIR': shared.opts.openvino_cache_path + '/blob'})
 
-    shared.compiled_model_state.compiling_vae = False
+    shared.compiled_model_state.compile_dont_use_4bit = False
     compiled_model = core.compile_model(om, get_device())
     return compiled_model
 
@@ -344,7 +344,11 @@ def partition_graph(gm: GraphModule, use_python_fusion_cache: bool, model_hash_s
 
 def generate_subgraph_str(tensor):
     if hasattr(tensor, "weight"):
-        shared.compiled_model_state.model_str = shared.compiled_model_state.model_str + str(tensor.weight)
+        shared.compiled_model_state.model_str = shared.compiled_model_state.model_str + sha256(str(tensor.weight).encode('utf-8')).hexdigest()
+    return tensor
+
+def get_subgraph_type(tensor):
+    shared.compiled_model_state.subgraph_type.append(type(tensor))
     return tensor
 
 @register_backend
@@ -353,12 +357,28 @@ def openvino_fx(subgraph, example_inputs):
     executor_parameters = None
     inputs_reversed = False
     maybe_fs_cached_name = None
+
+    shared.compiled_model_state.subgraph_type = []
+    subgraph.apply(get_subgraph_type)
+
+    # SD 1.5 / SDXL VAE
+    if (shared.compiled_model_state.subgraph_type[0] is torch.nn.modules.conv.Conv2d and
+        shared.compiled_model_state.subgraph_type[1] is torch.nn.modules.conv.Conv2d and
+        shared.compiled_model_state.subgraph_type[2] is torch.nn.modules.normalization.GroupNorm and
+        shared.compiled_model_state.subgraph_type[3] is torch.nn.modules.activation.SiLU):
+
+        shared.compiled_model_state.compile_dont_use_4bit = True
+
     if not shared.opts.openvino_disable_model_caching:
         os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "1")
-        # Create a hash to be used for caching
         shared.compiled_model_state.model_str = ""
+
+        # Create a hash to be used for caching
         subgraph.apply(generate_subgraph_str)
+        shared.compiled_model_state.model_str = shared.compiled_model_state.model_str + sha256(subgraph.code.encode('utf-8')).hexdigest()
         model_hash_str = sha256(shared.compiled_model_state.model_str.encode('utf-8')).hexdigest()
+        shared.compiled_model_state.model_str = ""
+
         if (shared.compiled_model_state.cn_model != [] and shared.compiled_model_state.partition_id == 0):
             model_hash_str = model_hash_str + str(shared.compiled_model_state.cn_model)
 
@@ -383,9 +403,17 @@ def openvino_fx(subgraph, example_inputs):
                                 example_inputs_reordered.append(example_inputs[idx1])
                 example_inputs = example_inputs_reordered
 
-            # Delete unused subgraphs
-            subgraph = subgraph.apply(sd_models.convert_to_faketensors)
-            devices.torch_gc(force=True)
+            # SD 1.5 / SDXL Text Encoder
+            if (shared.compiled_model_state.subgraph_type[0] is torch.nn.modules.sparse.Embedding and
+                shared.compiled_model_state.subgraph_type[1] is torch.nn.modules.sparse.Embedding and
+                shared.compiled_model_state.subgraph_type[2] is torch.nn.modules.normalization.LayerNorm and
+                shared.compiled_model_state.subgraph_type[3] is torch.nn.modules.linear.Linear):
+
+                pass # Fails with FakeTensors or Downcast
+            else:
+                # Delete unused subgraphs
+                subgraph = subgraph.apply(sd_models.convert_to_faketensors)
+                devices.torch_gc(force=True)
 
             # Model is fully supported and already cached. Run the cached OV model directly.
             compiled_model = openvino_compile_cached_model(maybe_fs_cached_name, *example_inputs)

modules/modelloader.py

@@ -230,7 +230,7 @@ def load_diffusers_models(model_path: str, command_path: str = None, clear=True)
                 if not os.path.isfile(os.path.join(cache_path, "hidden")):
                     output.append(str(r.repo_id))
             """
-            for folder in os.listdir(place):
+            for folder in shared.listdir(place):
                 try:
                     if "--" not in folder:
                         continue
@@ -240,7 +240,7 @@ def load_diffusers_models(model_path: str, command_path: str = None, clear=True)
                     name = name.replace("--", "/")
                     folder = os.path.join(place, folder)
                     friendly = os.path.join(place, name)
-                    snapshots = os.listdir(os.path.join(folder, "snapshots"))
+                    snapshots = shared.listdir(os.path.join(folder, "snapshots"))
                     if len(snapshots) == 0:
                         shared.log.warning(f"Diffusers folder has no snapshots: location={place} folder={folder} name={name}")
                         continue
@@ -450,7 +450,7 @@ def move_files(src_path: str, dest_path: str, ext_filter: str = None):
         if not os.path.exists(dest_path):
             os.makedirs(dest_path)
         if os.path.exists(src_path):
-            for file in os.listdir(src_path):
+            for file in shared.listdir(src_path):
                 fullpath = os.path.join(src_path, file)
                 if os.path.isfile(fullpath):
                     if ext_filter is not None:

modules/postprocessing.py

@@ -38,7 +38,7 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp
     elif extras_mode == 2:
         assert not shared.cmd_opts.hide_ui_dir_config, '--hide-ui-dir-config option must be disabled'
         assert input_dir, 'input directory not selected'
-        image_list = shared.listfiles(input_dir)
+        image_list = shared.listdir(input_dir)
         for filename in image_list:
             try:
                 image = Image.open(filename)

modules/processing_vae.py

@@ -48,13 +48,9 @@ def full_vae_decode(latents, model):
         model.upcast_vae()
         latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype)
 
-    # OpenVINO with INT4 doesn't work with VAE decode so we pass that we are using VAE right now to OpenVINO
-    if shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx" and shared.compiled_model_state.first_pass_vae:
-        shared.compiled_model_state.compiling_vae = True
-
     decoded = model.vae.decode(latents / model.vae.config.scaling_factor, return_dict=False)[0]
 
-    # Downcast VAE after OpenVINO compile
+    # Delete PyTorch VAE after OpenVINO compile
     if shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx" and shared.compiled_model_state.first_pass_vae:
         shared.compiled_model_state.first_pass_vae = False
         if hasattr(shared.sd_model, "vae"):

modules/script_loading.py

@@ -7,6 +7,7 @@ from installer import setup_logging, args
 
 
 preloaded = []
+debug = os.environ.get('SD_SCRIPT_DEBUG', None)
 
 
 def load_module(path):
@@ -20,9 +21,12 @@ def load_module(path):
         if '/sd-extension-' in path: # safe extensions without stdout intercept
             module_spec.loader.exec_module(module)
         else:
-            # stdout = io.StringIO()
-            with contextlib.redirect_stdout(io.StringIO()) as stdout:
+            if debug:
                 module_spec.loader.exec_module(module)
+                stdout = io.StringIO()
+            else:
+                with contextlib.redirect_stdout(io.StringIO()) as stdout:
+                    module_spec.loader.exec_module(module)
             setup_logging() # reset since scripts can hijaack logging
             for line in stdout.getvalue().splitlines():
                 if len(line) > 0:

modules/sd_hijack_hypertile.py

@@ -190,7 +190,7 @@ def context_hypertile_vae(p):
         # shared.log.warning('Hypertile VAE is enabled but no VAE model was found')
         return nullcontext()
     else:
-        tile_size = shared.opts.hypertile_vae_tile if shared.opts.hypertile_vae_tile > 0 else max(256, 64 * min(p.width // 128, p.height // 128))
+        tile_size = shared.opts.hypertile_vae_tile if shared.opts.hypertile_vae_tile > 0 else max(128, 64 * min(p.width // 128, p.height // 128))
         shared.log.info(f'Applying hypertile: vae={tile_size}')
         p.extra_generation_params['Hypertile VAE'] = tile_size
         return split_attention(vae, tile_size=tile_size, min_tile_size=128, swap_size=1)
@@ -216,7 +216,7 @@ def context_hypertile_unet(p):
         # shared.log.warning('Hypertile UNet is enabled but no Unet model was found')
         return nullcontext()
     else:
-        tile_size = shared.opts.hypertile_unet_tile if shared.opts.hypertile_unet_tile > 0 else max(256, 64 * min(p.width // 128, p.height // 128))
+        tile_size = shared.opts.hypertile_unet_tile if shared.opts.hypertile_unet_tile > 0 else max(128, 64 * min(p.width // 128, p.height // 128))
         shared.log.info(f'Applying hypertile: unet={tile_size}')
         p.extra_generation_params['Hypertile UNet'] = tile_size
         return split_attention(unet, tile_size=tile_size, min_tile_size=128, swap_size=1)

modules/sd_models.py

@@ -1036,6 +1036,10 @@ def set_diffuser_pipe(pipe, new_pipe_type):
     image_encoder = getattr(pipe, "image_encoder", None)
     feature_extractor = getattr(pipe, "feature_extractor", None)
 
+    # skip specific pipelines
+    if pipe.__class__.__name__ == 'StableDiffusionReferencePipeline' or pipe.__class__.__name__ == 'StableDiffusionAdapterPipeline':
+        return pipe
+
     try:
         if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE:
             new_pipe = diffusers.AutoPipelineForText2Image.from_pipe(pipe)
@@ -1044,7 +1048,7 @@ def set_diffuser_pipe(pipe, new_pipe_type):
         elif new_pipe_type == DiffusersTaskType.INPAINTING:
             new_pipe = diffusers.AutoPipelineForInpainting.from_pipe(pipe)
     except Exception as e: # pylint: disable=unused-variable
-        shared.log.error(f'Failed to change: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}')
+        shared.log.warning(f'Failed to change: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}')
         return pipe
 
     if pipe.__class__ == new_pipe.__class__:
@@ -1263,10 +1267,10 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model')
 
 
 def convert_to_faketensors(tensor):
-        fake = torch._subclasses.fake_tensor.FakeTensorMode()
-        if hasattr(tensor, "weight"):
-            tensor.weight = torch.nn.Parameter(fake.from_tensor(tensor.weight))
-        return tensor
+    fake_module = torch._subclasses.fake_tensor.FakeTensorMode(allow_non_fake_inputs=True) # pylint: disable=protected-access
+    if hasattr(tensor, "weight"):
+        tensor.weight = torch.nn.Parameter(fake_module.from_tensor(tensor.weight))
+    return tensor
 
 
 def disable_offload(sd_model):
@@ -1280,6 +1284,9 @@ def disable_offload(sd_model):
 
 
 def unload_model_weights(op='model', change_from='none'):
+    if shared.compiled_model_state is not None:
+        shared.compiled_model_state.compiled_cache.clear()
+        shared.compiled_model_state.partitioned_modules.clear()
     if op == 'model' or op == 'dict':
         if model_data.sd_model:
             if (shared.backend == shared.Backend.ORIGINAL and change_from != shared.Backend.DIFFUSERS) or change_from == shared.Backend.ORIGINAL:

modules/sd_models_compile.py

@@ -21,7 +21,8 @@ class CompiledModelState:
         self.lora_compile = False
         self.compiled_cache = {}
         self.partitioned_modules = {}
-        self.compiling_vae = False
+        self.subgraph_type = []
+        self.compile_dont_use_4bit = False
 
 
 def ipex_optimize(sd_model):
@@ -72,13 +73,13 @@ def optimize_openvino():
         if shared.compiled_model_state is None:
             shared.compiled_model_state = CompiledModelState()
         else:
-            if not shared.compiled_model_state.lora_compile:
-                shared.compiled_model_state.lora_compile = False
-                shared.compiled_model_state.lora_model = []
             shared.compiled_model_state.compiled_cache.clear()
             shared.compiled_model_state.partitioned_modules.clear()
-            shared.compiled_model_state.partition_id = 0
-            shared.compiled_model_state.model_str = ""
+            backup_lora_model = []
+            if shared.compiled_model_state.lora_compile:
+                backup_lora_model = shared.compiled_model_state.lora_model
+            shared.compiled_model_state = CompiledModelState()
+            shared.compiled_model_state.lora_model = backup_lora_model
         shared.compiled_model_state.first_pass = True if not shared.opts.cuda_compile_precompile else False
         shared.compiled_model_state.first_pass_vae = True if not shared.opts.cuda_compile_precompile else False
         shared.compiled_model_state.first_pass_refiner = True if not shared.opts.cuda_compile_precompile else False
@@ -149,10 +150,10 @@ def compile_torch(sd_model):
 
         t0 = time.time()
         if shared.opts.cuda_compile:
-            if shared.opts.cuda_compile and (not hasattr(sd_model, 'unet') or not hasattr(sd_model.unet, 'config')):
-                shared.log.warning('Model compile enabled but model has no Unet')
-            else:
+            if hasattr(sd_model, 'unet') and hasattr(sd_model.unet, 'config'):
                 sd_model.unet = torch.compile(sd_model.unet, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph)
+            else:
+                shared.log.warning('Model compile enabled but model has no Unet')
         if shared.opts.cuda_compile_vae:
             if hasattr(sd_model, 'vae') and hasattr(sd_model.vae, 'decode'):
                 sd_model.vae.decode = torch.compile(sd_model.vae.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph)
@@ -160,6 +161,13 @@ def compile_torch(sd_model):
                 sd_model.movq.decode = torch.compile(sd_model.movq.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph)
             else:
                 shared.log.warning('Model compile enabled but model has no VAE')
+        if shared.opts.cuda_compile_text_encoder:
+            if hasattr(sd_model, 'text_encoder'):
+                sd_model.text_encoder = torch.compile(sd_model.text_encoder, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph)
+                if hasattr(sd_model, 'text_encoder_2'):
+                    sd_model.text_encoder_2 = torch.compile(sd_model.text_encoder_2, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph)
+            else:
+                shared.log.warning('Text Encoder compile enabled but model has no Text Encoder')
         setup_logging() # compile messes with logging so reset is needed
         if shared.opts.cuda_compile_precompile:
             sd_model("dummy prompt")

modules/shared.py

@@ -70,6 +70,8 @@ restricted_opts = {
 resize_modes = ["None", "Fixed", "Crop", "Fill", "Latent"]
 compatibility_opts = ['clip_skip', 'uni_pc_lower_order_final', 'uni_pc_order']
 console = Console(log_time=True, log_time_format='%H:%M:%S-%f')
+dir_timestamps = {}
+dir_cache = {}
 
 
 class Backend(Enum):
@@ -327,7 +329,8 @@ options_templates.update(options_section(('cuda', "Compute Settings"), {
     "cuda_compile_sep": OptionInfo("<h2>Model Compile</h2>", "", gr.HTML),
     "cuda_compile": OptionInfo(False if not cmd_opts.use_openvino else True, "Compile UNet"),
     "cuda_compile_vae": OptionInfo(False if not cmd_opts.use_openvino else True, "Compile VAE"),
-    "cuda_compile_upscaler": OptionInfo(False if not cmd_opts.use_openvino else True, "Compile upscaler"),
+    "cuda_compile_text_encoder": OptionInfo(False, "Compile Text Encoder"),
+    "cuda_compile_upscaler": OptionInfo(False if not cmd_opts.use_openvino else True, "Compile Upscaler"),
     "cuda_compile_backend": OptionInfo("none" if not cmd_opts.use_openvino else "openvino_fx", "Model compile backend", gr.Radio, {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex', 'openvino_fx', 'stable-fast']}),
     "cuda_compile_mode": OptionInfo("default", "Model compile mode", gr.Radio, {"choices": ['default', 'reduce-overhead', 'max-autotune', 'max-autotune-no-cudagraphs']}),
     "cuda_compile_fullgraph": OptionInfo(False, "Model compile fullgraph"),
@@ -887,9 +890,16 @@ def restore_defaults(restart=True):
     restart_server(restart)
 
 
-def listfiles(dirname):
-    filenames = [os.path.join(dirname, x) for x in sorted(os.listdir(dirname), key=str.lower) if not x.startswith(".")]
-    return [file for file in filenames if os.path.isfile(file)]
+def listdir(path):
+    if not os.path.exists(path):
+        return []
+    mtime = os.path.getmtime(path)
+    if path in dir_timestamps and mtime == dir_timestamps[path]:
+        return dir_cache[path]
+    else:
+        dir_cache[path] = [os.path.join(path, f) for f in os.listdir(path)]
+        dir_timestamps[path] = mtime
+        return dir_cache[path]
 
 
 def walk_files(path, allowed_extensions=None):

modules/textual_inversion/preprocess.py

@@ -25,10 +25,6 @@ def preprocess(id_task, process_src, process_dst, process_width, process_height,
             deepbooru.model.stop()
 
 
-def listfiles(dirname):
-    return os.listdir(dirname)
-
-
 class PreprocessParams:
     src = None
     dstdir = None
@@ -137,17 +133,12 @@ def preprocess_work(process_src, process_dst, process_width, process_height, pre
     dst = os.path.abspath(process_dst)
     split_threshold = max(0.0, min(1.0, split_threshold))
     overlap_ratio = max(0.0, min(0.9, overlap_ratio))
-
     assert src != dst, 'same directory specified as source and destination'
-
     os.makedirs(dst, exist_ok=True)
-
-    files = listfiles(src)
-
+    files = shared.listdir(src)
     shared.state.job = "preprocess"
     shared.state.textinfo = "Preprocessing..."
     shared.state.job_count = len(files)
-
     params = PreprocessParams()
     params.dstdir = dst
     params.flip = process_flip
@@ -155,7 +146,6 @@ def preprocess_work(process_src, process_dst, process_width, process_height, pre
     params.process_caption = process_caption
     params.process_caption_deepbooru = process_caption_deepbooru
     params.preprocess_txt_action = preprocess_txt_action
-
     pbar = tqdm(files)
     for index, imagefile in enumerate(pbar):
         params.subindex = 0
@@ -171,9 +161,7 @@ def preprocess_work(process_src, process_dst, process_width, process_height, pre
         description = f"Preprocessing image {index + 1}/{len(files)}"
         pbar.set_description(description)
         shared.state.textinfo = description
-
         params.src = filename
-
         existing_caption = None
         existing_caption_filename = f"{os.path.splitext(filename)[0]}.txt"
         if os.path.exists(existing_caption_filename):
@@ -181,32 +169,25 @@ def preprocess_work(process_src, process_dst, process_width, process_height, pre
                 existing_caption = file.read()
         else:
             existing_caption_filename = None
-
         if shared.state.interrupted:
             break
-
         if img.height > img.width:
             ratio = (img.width * height) / (img.height * width)
             inverse_xy = False
         else:
             ratio = (img.height * width) / (img.width * height)
             inverse_xy = True
-
         process_default_resize = True
-
         if process_split and ratio < 1.0 and ratio <= split_threshold:
             for splitted in split_pic(img, inverse_xy, width, height, overlap_ratio):
                 save_pic(splitted, index, params, existing_caption=existing_caption, existing_caption_filename=existing_caption_filename)
             process_default_resize = False
-
         if process_focal_crop and img.height != img.width:
-
             dnn_model_path = None
             try:
                 dnn_model_path = autocrop.download_and_cache_models(os.path.join(paths.models_path, "opencv"))
             except Exception as e:
                 print("Unable to load face detection model for auto crop selection. Falling back to lower quality haar method.", e)
-
             autocrop_settings = autocrop.Settings(
                 crop_width = width,
                 crop_height = height,
@@ -227,13 +208,10 @@ def preprocess_work(process_src, process_dst, process_width, process_height, pre
             else:
                 print(f"skipped {img.width}x{img.height} image {filename} (can't find suitable size within error threshold)")
             process_default_resize = False
-
         if process_keep_original_size:
             save_pic(img, index, params, existing_caption=existing_caption)
             process_default_resize = False
-
         if process_default_resize:
             img = images.resize_image(1, img, width, height)
             save_pic(img, index, params, existing_caption=existing_caption)
-
         shared.state.nextjob()

modules/ui_control.py

@@ -421,20 +421,19 @@ def create_ui(_blocks: gr.Blocks=None):
                         num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1)
                     controlnet_ui_units = [] # list of hidable accordions
                     for i in range(max_units):
-                        with gr.Accordion(f'Control unit {i+1}', visible= i < num_controlnet_units.value) as unit_ui:
+                        with gr.Accordion(f'Control unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui:
                             with gr.Row():
-                                with gr.Column():
-                                    with gr.Row():
-                                        enabled_cb = gr.Checkbox(value= i==0, label="")
-                                        process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
-                                        model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None')
-                                        ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, 'refresh_controlnet_models')
-                                        model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10)
-                                        control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0)
-                                        control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0)
-                                        reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
-                                        image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
-                                        process_btn= ui_components.ToolButton(value=ui_symbols.preview)
+                                enabled_cb = gr.Checkbox(value= i==0, label="")
+                                process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
+                                model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None')
+                                ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, 'refresh_controlnet_models')
+                                model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10)
+                                control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0)
+                                control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0)
+                                reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
+                                image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
+                                process_btn= ui_components.ToolButton(value=ui_symbols.preview)
+                                image_preview = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=False, height=128, width=128, visible=False)
                         controlnet_ui_units.append(unit_ui)
                         units.append(unit.Unit(
                             unit_type = 'controlnet',
@@ -448,6 +447,7 @@ def create_ui(_blocks: gr.Blocks=None):
                             preview_process = preview_process,
                             preview_btn = process_btn,
                             image_upload = image_upload,
+                            image_preview = image_preview,
                             control_start = control_start,
                             control_end = control_end,
                             extra_controls = extra_controls,
@@ -457,51 +457,6 @@ def create_ui(_blocks: gr.Blocks=None):
                             units[-1].enabled = True # enable first unit in group
                     num_controlnet_units.change(fn=display_units, inputs=[num_controlnet_units], outputs=controlnet_ui_units)
 
-                with gr.Tab('XS') as _tab_controlnetxs:
-                    gr.HTML('<a href="https://vislearn.github.io/ControlNet-XS/">ControlNet XS</a>')
-                    with gr.Row():
-                        extra_controls = [
-                            gr.Slider(label="Time embedding mix", minimum=0.0, maximum=1.0, step=0.05, value=0.0, scale=3)
-                        ]
-                        num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1)
-                    controlnetxs_ui_units = [] # list of hidable accordions
-                    for i in range(max_units):
-                        with gr.Accordion(f'Control unit {i+1}', visible= i < num_controlnet_units.value) as unit_ui:
-                            with gr.Row():
-                                with gr.Column():
-                                    with gr.Row():
-                                        enabled_cb = gr.Checkbox(value= i==0, label="")
-                                        process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
-                                        model_id = gr.Dropdown(label="ControlNet-XS", choices=xs.list_models(), value='None')
-                                        ui_common.create_refresh_button(model_id, xs.list_models, lambda: {"choices": xs.list_models(refresh=True)}, 'refresh_xs_models')
-                                        model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10)
-                                        control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0)
-                                        control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0)
-                                        reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
-                                        image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
-                                        process_btn= ui_components.ToolButton(value=ui_symbols.preview)
-                        controlnetxs_ui_units.append(unit_ui)
-                        units.append(unit.Unit(
-                            unit_type = 'xs',
-                            result_txt = result_txt,
-                            image_input = input_image,
-                            enabled_cb = enabled_cb,
-                            reset_btn = reset_btn,
-                            process_id = process_id,
-                            model_id = model_id,
-                            model_strength = model_strength,
-                            preview_process = preview_process,
-                            preview_btn = process_btn,
-                            image_upload = image_upload,
-                            control_start = control_start,
-                            control_end = control_end,
-                            extra_controls = extra_controls,
-                            )
-                        )
-                        if i == 0:
-                            units[-1].enabled = True # enable first unit in group
-                    num_controlnet_units.change(fn=display_units, inputs=[num_controlnet_units], outputs=controlnetxs_ui_units)
-
                 with gr.Tab('Adapter') as _tab_adapter:
                     gr.HTML('<a href="https://github.com/TencentARC/T2I-Adapter">T2I-Adapter</a>')
                     with gr.Row():
@@ -511,18 +466,17 @@ def create_ui(_blocks: gr.Blocks=None):
                         num_adapter_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1)
                     adapter_ui_units = [] # list of hidable accordions
                     for i in range(max_units):
-                        with gr.Accordion(f'Adapter unit {i+1}', visible= i < num_adapter_units.value) as unit_ui:
+                        with gr.Accordion(f'Adapter unit {i+1}', visible= i < num_adapter_units.value, elem_classes='control-unit') as unit_ui:
                             with gr.Row():
-                                with gr.Column():
-                                    with gr.Row():
-                                        enabled_cb = gr.Checkbox(value= i == 0, label="Enabled")
-                                        process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
-                                        model_id = gr.Dropdown(label="Adapter", choices=t2iadapter.list_models(), value='None')
-                                        ui_common.create_refresh_button(model_id, t2iadapter.list_models, lambda: {"choices": t2iadapter.list_models(refresh=True)}, 'refresh_adapter_models')
-                                        model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10)
-                                        reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
-                                        image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
-                                        process_btn= ui_components.ToolButton(value=ui_symbols.preview)
+                                enabled_cb = gr.Checkbox(value= i == 0, label="Enabled")
+                                process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
+                                model_id = gr.Dropdown(label="Adapter", choices=t2iadapter.list_models(), value='None')
+                                ui_common.create_refresh_button(model_id, t2iadapter.list_models, lambda: {"choices": t2iadapter.list_models(refresh=True)}, 'refresh_adapter_models')
+                                model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10)
+                                reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
+                                image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
+                                process_btn= ui_components.ToolButton(value=ui_symbols.preview)
+                                image_preview = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=False, height=128, width=128, visible=False)
                         adapter_ui_units.append(unit_ui)
                         units.append(unit.Unit(
                             unit_type = 'adapter',
@@ -536,6 +490,7 @@ def create_ui(_blocks: gr.Blocks=None):
                             preview_process = preview_process,
                             preview_btn = process_btn,
                             image_upload = image_upload,
+                            image_preview = image_preview,
                             extra_controls = extra_controls,
                             )
                         )
@@ -543,6 +498,51 @@ def create_ui(_blocks: gr.Blocks=None):
                             units[-1].enabled = True # enable first unit in group
                     num_adapter_units.change(fn=display_units, inputs=[num_adapter_units], outputs=adapter_ui_units)
 
+                with gr.Tab('XS') as _tab_controlnetxs:
+                    gr.HTML('<a href="https://vislearn.github.io/ControlNet-XS/">ControlNet XS</a>')
+                    with gr.Row():
+                        extra_controls = [
+                            gr.Slider(label="Time embedding mix", minimum=0.0, maximum=1.0, step=0.05, value=0.0, scale=3)
+                        ]
+                        num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1)
+                    controlnetxs_ui_units = [] # list of hidable accordions
+                    for i in range(max_units):
+                        with gr.Accordion(f'Control unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui:
+                            with gr.Row():
+                                enabled_cb = gr.Checkbox(value= i==0, label="")
+                                process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
+                                model_id = gr.Dropdown(label="ControlNet-XS", choices=xs.list_models(), value='None')
+                                ui_common.create_refresh_button(model_id, xs.list_models, lambda: {"choices": xs.list_models(refresh=True)}, 'refresh_xs_models')
+                                model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10)
+                                control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0)
+                                control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0)
+                                reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
+                                image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
+                                process_btn= ui_components.ToolButton(value=ui_symbols.preview)
+                                image_preview = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=False, height=128, width=128, visible=False)
+                        controlnetxs_ui_units.append(unit_ui)
+                        units.append(unit.Unit(
+                            unit_type = 'xs',
+                            result_txt = result_txt,
+                            image_input = input_image,
+                            enabled_cb = enabled_cb,
+                            reset_btn = reset_btn,
+                            process_id = process_id,
+                            model_id = model_id,
+                            model_strength = model_strength,
+                            preview_process = preview_process,
+                            preview_btn = process_btn,
+                            image_upload = image_upload,
+                            image_preview = image_preview,
+                            control_start = control_start,
+                            control_end = control_end,
+                            extra_controls = extra_controls,
+                            )
+                        )
+                        if i == 0:
+                            units[-1].enabled = True # enable first unit in group
+                    num_controlnet_units.change(fn=display_units, inputs=[num_controlnet_units], outputs=controlnetxs_ui_units)
+
                 with gr.Tab('Lite') as _tab_lite:
                     gr.HTML('<a href="https://huggingface.co/kohya-ss/controlnet-lllite">Control LLLite</a>')
                     with gr.Row():
@@ -551,18 +551,17 @@ def create_ui(_blocks: gr.Blocks=None):
                         num_lite_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1)
                     lite_ui_units = [] # list of hidable accordions
                     for i in range(max_units):
-                        with gr.Accordion(f'Control unit {i+1}', visible= i < num_lite_units.value) as unit_ui:
+                        with gr.Accordion(f'Control unit {i+1}', visible= i < num_lite_units.value, elem_classes='control-unit') as unit_ui:
                             with gr.Row():
-                                with gr.Column():
-                                    with gr.Row():
-                                        enabled_cb = gr.Checkbox(value= i == 0, label="Enabled")
-                                        process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
-                                        model_id = gr.Dropdown(label="Model", choices=lite.list_models(), value='None')
-                                        ui_common.create_refresh_button(model_id, lite.list_models, lambda: {"choices": lite.list_models(refresh=True)}, 'refresh_lite_models')
-                                        model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10)
-                                        reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
-                                        image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
-                                        process_btn= ui_components.ToolButton(value=ui_symbols.preview)
+                                enabled_cb = gr.Checkbox(value= i == 0, label="Enabled")
+                                process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
+                                model_id = gr.Dropdown(label="Model", choices=lite.list_models(), value='None')
+                                ui_common.create_refresh_button(model_id, lite.list_models, lambda: {"choices": lite.list_models(refresh=True)}, 'refresh_lite_models')
+                                model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10)
+                                reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
+                                image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
+                                image_preview = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=False, height=128, width=128, visible=False)
+                                process_btn= ui_components.ToolButton(value=ui_symbols.preview)
                         lite_ui_units.append(unit_ui)
                         units.append(unit.Unit(
                             unit_type = 'lite',
@@ -576,6 +575,7 @@ def create_ui(_blocks: gr.Blocks=None):
                             preview_process = preview_process,
                             preview_btn = process_btn,
                             image_upload = image_upload,
+                            image_preview = image_preview,
                             extra_controls = extra_controls,
                             )
                         )
@@ -593,16 +593,15 @@ def create_ui(_blocks: gr.Blocks=None):
                             gr.Slider(label="Reference adain weight", minimum=0.0, maximum=2.0, step=0.05, value=1.0, interactive=True),
                         ]
                     for i in range(1): # can only have one reference unit
-                        with gr.Accordion(f'Reference unit {i+1}', visible=True) as unit_ui:
+                        with gr.Accordion(f'Reference unit {i+1}', visible=True, elem_classes='control-unit') as unit_ui:
                             with gr.Row():
-                                with gr.Column():
-                                    with gr.Row():
-                                        enabled_cb = gr.Checkbox(value= i == 0, label="Enabled", visible=False)
-                                        model_id = gr.Dropdown(label="Reference", choices=reference.list_models(), value='Reference', visible=False)
-                                        model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, visible=False)
-                                        reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
-                                        image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
-                                        process_btn= ui_components.ToolButton(value=ui_symbols.preview)
+                                enabled_cb = gr.Checkbox(value= i == 0, label="Enabled", visible=False)
+                                model_id = gr.Dropdown(label="Reference", choices=reference.list_models(), value='Reference', visible=False)
+                                model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, visible=False)
+                                reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
+                                image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
+                                image_preview = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=False, height=128, width=128, visible=False)
+                                process_btn= ui_components.ToolButton(value=ui_symbols.preview)
                         units.append(unit.Unit(
                             unit_type = 'reference',
                             result_txt = result_txt,
@@ -615,6 +614,7 @@ def create_ui(_blocks: gr.Blocks=None):
                             preview_process = preview_process,
                             preview_btn = process_btn,
                             image_upload = image_upload,
+                            image_preview = image_preview,
                             extra_controls = extra_controls,
                             )
                         )

modules/ui_img2img.py

@@ -20,7 +20,7 @@ def process_interrogate(interrogation_function, mode, ii_input_files, ii_input_d
             if not os.path.isdir(ii_input_dir):
                 shared.log.error(f"Interrogate: Input directory not found: {ii_input_dir}")
                 return [gr.update(), None]
-            images = shared.listfiles(ii_input_dir)
+            images = shared.listdir(ii_input_dir)
         if ii_output_dir != "":
             os.makedirs(ii_output_dir, exist_ok=True)
         else:

modules/ui_interrogate.py

@@ -119,7 +119,7 @@ def batch_process(batch_files, batch_folder, batch_str, mode, clip_model, write)
     if batch_folder is not None:
         files += [f.name for f in batch_folder]
     if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str):
-        files += [os.path.join(batch_str, f) for f in os.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))]
+        files += [os.path.join(batch_str, f) for f in shared.listdir(batch_str) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))]
     if len(files) == 0:
         shared.log.error('Interrogate batch no images')
         return ''

modules/upscaler.py

@@ -52,7 +52,7 @@ class Upscaler:
             pass
 
     def find_folder(self, folder, scalers, loaded):
-        for fn in os.listdir(folder): # from folder
+        for fn in modules.shared.listdir(folder): # from folder
             file_name = os.path.join(folder, fn)
             if os.path.isdir(file_name):
                 self.find_folder(file_name, scalers, loaded)
@@ -83,19 +83,6 @@ class Upscaler:
         if not os.path.exists(self.user_path):
             return scalers
         self.find_folder(self.user_path, scalers, loaded)
-        """
-        for fn in os.listdir(self.user_path): # from folder
-            if not fn.endswith('.pth') and not fn.endswith('.pt'):
-                continue
-            file_name = os.path.join(self.user_path, fn)
-            if file_name not in loaded:
-                model_name = os.path.splitext(fn)[0]
-                scaler = UpscalerData(name=f'{self.name} {model_name}', path=file_name, upscaler=self)
-                scaler.custom = True
-                scalers.append(scaler)
-                loaded.append(file_name)
-                # modules.shared.log.debug(f'Upscaler type={self.name} folder="{self.user_path}" model="{model_name}" path="{file_name}"')
-        """
         return scalers
 
     @abstractmethod

scripts/faceid.py

@@ -6,11 +6,30 @@ import gradio as gr
 import diffusers
 import huggingface_hub as hf
 from modules import scripts, processing, shared, devices
-from installer import installed
 
 
+MODELS = {
+    'FaceID Base': 'h94/IP-Adapter-FaceID/ip-adapter-faceid_sd15.bin',
+    'FaceID Plus': 'h94/IP-Adapter-FaceID/ip-adapter-faceid-plus_sd15.bin',
+    'FaceID Plus v2': 'h94/IP-Adapter-FaceID/ip-adapter-faceid-plusv2_sd15.bin',
+    'FaceID XL': 'h94/IP-Adapter-FaceID/ip-adapter-faceid_sdxl.bin'
+}
 app = None
-ok = installed('insightface', reload=False, quiet=True) and installed('ip_adapter', reload=False, quiet=True)
+ip_model = None
+ip_model_name = None
+ip_model_tokens = None
+ip_model_rank = None
+
+
+def dependencies():
+    from installer import installed, install
+    packages = [
+        ('insightface', 'insightface'),
+        ('git+https://github.com/tencent-ailab/IP-Adapter.git', 'ip_adapter'),
+    ]
+    for pkg in packages:
+        if not installed(pkg[1], reload=False, quiet=True):
+            install(pkg[0], pkg[1], ignore=True)
 
 
 class Script(scripts.Script):
@@ -18,32 +37,43 @@ class Script(scripts.Script):
         return 'FaceID'
 
     def show(self, is_img2img):
-        return ok if shared.backend == shared.Backend.DIFFUSERS else False
+        return True if shared.backend == shared.Backend.DIFFUSERS else False
 
     # return signature is array of gradio components
     def ui(self, _is_img2img):
         with gr.Row():
-            scale = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=1.0)
+            model = gr.Dropdown(choices=list(MODELS), label='Model', value='FaceID Base')
+        with gr.Row(visible=True):
+            override = gr.Checkbox(label='Override sampler', value=True)
+            cache = gr.Checkbox(label='Cache model', value=True)
+        with gr.Row(visible=True):
+            scale = gr.Slider(label='Strength', minimum=0.0, maximum=1.0, step=0.01, value=1.0)
+            structure = gr.Slider(label='Structure', minimum=0.0, maximum=1.0, step=0.01, value=1.0)
+        with gr.Row(visible=False):
+            rank = gr.Slider(label='Rank', minimum=4, maximum=256, step=4, value=128)
+            tokens = gr.Slider(label='Tokens', minimum=1, maximum=16, step=1, value=4)
         with gr.Row():
             image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512)
-        return [scale, image]
+        return [model, scale, image, override, rank, tokens, structure, cache]
 
-    def run(self, p: processing.StableDiffusionProcessing, scale, image): # pylint: disable=arguments-differ, unused-argument
+    def run(self, p: processing.StableDiffusionProcessing, model, scale, image, override, rank, tokens, structure, cache): # pylint: disable=arguments-differ, unused-argument
+        dependencies()
         try:
             import onnxruntime
             from insightface.app import FaceAnalysis
-            from ip_adapter.ip_adapter_faceid import IPAdapterFaceID
+            from insightface.utils import face_align
+            from ip_adapter.ip_adapter_faceid import IPAdapterFaceID, IPAdapterFaceIDPlus, IPAdapterFaceIDXL
         except Exception as e:
             shared.log.error(f'FaceID: {e}')
             return None
         if image is None:
             shared.log.error('FaceID: no init_images')
             return None
-        if shared.sd_model_type != 'sd':
+        if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
             shared.log.error('FaceID: base model not supported')
             return None
 
-        global app # pylint: disable=global-statement
+        global app, ip_model, ip_model_name, ip_model_tokens, ip_model_rank # pylint: disable=global-statement
         if app is None:
             shared.log.debug(f"ONNX: device={onnxruntime.get_device()} providers={onnxruntime.get_available_providers()}")
             app = FaceAnalysis(name="buffalo_l", providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
@@ -57,28 +87,68 @@ class Script(scripts.Script):
             return None
         for face in faces:
             shared.log.debug(f'FaceID face: score={face.det_score:.2f} gender={"female" if face.gender==0 else "male"} age={face.age} bbox={face.bbox}')
-        embeds = torch.from_numpy(faces[0].normed_embedding).unsqueeze(0)
+        face_embeds = torch.from_numpy(faces[0].normed_embedding).unsqueeze(0)
+        face_image = face_align.norm_crop(image, landmark=faces[0].kps, image_size=224) # you can also segment the face
 
-        ip_ckpt = "h94/IP-Adapter-FaceID/ip-adapter-faceid_sd15.bin"
-        shared.log.debug(f'FaceID model load: {ip_ckpt}')
+        ip_ckpt = MODELS[model]
         folder, filename = os.path.split(ip_ckpt)
         basename, _ext = os.path.splitext(filename)
         model_path = hf.hf_hub_download(repo_id=folder, filename=filename, cache_dir=shared.opts.diffusers_dir)
         if model_path is None:
-            shared.log.error(f'FaceID: model download failed: {ip_ckpt}')
+            shared.log.error(f'FaceID download failed: model={model} file={ip_ckpt}')
             return None
 
         processing.process_init(p)
-        shared.sd_model.scheduler = diffusers.DDIMScheduler(
-            num_train_timesteps=1000,
-            beta_start=0.00085,
-            beta_end=0.012,
-            beta_schedule="scaled_linear",
-            clip_sample=False,
-            set_alpha_to_one=False,
-            steps_offset=1,
-        )
-        ip_model = IPAdapterFaceID(shared.sd_model, model_path, devices.device)
+        if override:
+            shared.sd_model.scheduler = diffusers.DDIMScheduler(
+                num_train_timesteps=1000,
+                beta_start=0.00085,
+                beta_end=0.012,
+                beta_schedule="scaled_linear",
+                clip_sample=False,
+                set_alpha_to_one=False,
+                steps_offset=1,
+            )
+        shortcut = None
+        if ip_model is None or ip_model_name != model or ip_model_tokens != tokens or ip_model_rank != rank or not cache:
+            shared.log.debug(f'FaceID load: model={model} file={ip_ckpt} tokens={tokens} rank={rank}')
+            if 'Plus' in model:
+                image_encoder_path = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K"
+                ip_model = IPAdapterFaceIDPlus(
+                    sd_pipe=shared.sd_model,
+                    image_encoder_path=image_encoder_path,
+                    ip_ckpt=model_path,
+                    lora_rank=rank,
+                    num_tokens=tokens,
+                    device=devices.device,
+                    torch_dtype=devices.dtype,
+                )
+                shortcut = 'v2' in model
+            elif 'XL' in model:
+                ip_model = IPAdapterFaceIDXL(
+                    sd_pipe=shared.sd_model,
+                    ip_ckpt=model_path,
+                    lora_rank=rank,
+                    num_tokens=tokens,
+                    device=devices.device,
+                    torch_dtype=devices.dtype,
+                )
+            else:
+                ip_model = IPAdapterFaceID(
+                    sd_pipe=shared.sd_model,
+                    ip_ckpt=model_path,
+                    lora_rank=rank,
+                    num_tokens=tokens,
+                    device=devices.device,
+                    torch_dtype=devices.dtype,
+                )
+            ip_model_name = model
+            ip_model_tokens = tokens
+            ip_model_rank = rank
+        else:
+            shared.log.debug(f'FaceID cached: model={model} file={ip_ckpt} tokens={tokens} rank={rank}')
+
+        # main generate dict
         ip_model_dict = {
             'prompt': p.all_prompts[0],
             'negative_prompt': p.all_negative_prompts[0],
@@ -89,18 +159,34 @@ class Script(scripts.Script):
             'scale': scale,
             'guidance_scale': p.cfg_scale,
             'seed': int(p.all_seeds[0]),
-            'faceid_embeds': None,
+            'faceid_embeds': face_embeds.shape,
         }
+
+        # optional generate dict
+        if shortcut is not None:
+            ip_model_dict['shortcut'] = shortcut
+        if 'Plus' in model:
+            ip_model_dict['s_scale'] = structure
+            ip_model_dict['face_image'] = face_image.shape
         shared.log.debug(f'FaceID args: {ip_model_dict}')
-        ip_model_dict['faceid_embeds'] = embeds
+        if 'Plus' in model:
+            ip_model_dict['face_image'] = face_image
+        ip_model_dict['faceid_embeds'] = face_embeds
+
+        # run generate
         images = ip_model.generate(**ip_model_dict)
 
-        ip_model = None
+        if not cache:
+            ip_model = None
+            ip_model_name = None
+        devices.torch_gc()
+
         p.extra_generation_params["IP Adapter"] = f'{basename}:{scale}'
         for i, face in enumerate(faces):
             p.extra_generation_params[f"FaceID {i} score"] = f'{face.det_score:.2f}'
             p.extra_generation_params[f"FaceID {i} gender"] = "female" if face.gender==0 else "male"
             p.extra_generation_params[f"FaceID {i} age"] = face.age
+
         processed = processing.Processed(
             p,
             images_list=images,
@@ -110,5 +196,4 @@ class Script(scripts.Script):
         )
         processed.info = processed.infotext(p, 0)
         processed.infotexts = [processed.info]
-        devices.torch_gc()
         return processed

scripts/ipadapter.py

@@ -15,20 +15,22 @@ from modules import scripts, processing, shared, devices
 
 image_encoder = None
 image_encoder_type = None
+image_encoder_name = None
 loaded = None
 checkpoint = None
+base_repo = "h94/IP-Adapter"
 ADAPTERS = {
     'None': 'none',
-    'Base': 'ip-adapter_sd15',
-    'Light': 'ip-adapter_sd15_light',
-    'Plus': 'ip-adapter-plus_sd15',
-    'Plus Face': 'ip-adapter-plus-face_sd15',
-    'Full face': 'ip-adapter-full-face_sd15',
-    'Base SXDL': 'ip-adapter_sdxl',
-    # 'models/ip-adapter_sd15_vit-G', # RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x1024 and 1280x3072)
-    # 'sdxl_models/ip-adapter_sdxl_vit-h',
-    # 'sdxl_models/ip-adapter-plus_sdxl_vit-h',
-    # 'sdxl_models/ip-adapter-plus-face_sdxl_vit-h',
+    'Base': 'ip-adapter_sd15.safetensors',
+    'Base ViT-G': 'ip-adapter_sd15_vit-G.safetensors',
+    'Light': 'ip-adapter_sd15_light.safetensors',
+    'Plus': 'ip-adapter-plus_sd15.safetensors',
+    'Plus Face': 'ip-adapter-plus-face_sd15.safetensors',
+    'Full Face': 'ip-adapter-full-face_sd15.safetensors',
+    'Base SXDL': 'ip-adapter_sdxl.safetensors',
+    'Base ViT-H SXDL': 'ip-adapter_sdxl_vit-h.safetensors',
+    'Plus ViT-H SXDL': 'ip-adapter-plus_sdxl_vit-h.safetensors',
+    'Plus Face ViT-H SXDL': 'ip-adapter-plus-face_sdxl_vit-h.safetensors',
 }
 
 
@@ -48,9 +50,9 @@ class Script(scripts.Script):
                 image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512)
         return [adapter, scale, image]
 
-    def process(self, p: processing.StableDiffusionProcessing, adapter, scale, image): # pylint: disable=arguments-differ
+    def process(self, p: processing.StableDiffusionProcessing, adapter_name, scale, image): # pylint: disable=arguments-differ
         # overrides
-        adapter = ADAPTERS.get(adapter, None)
+        adapter = ADAPTERS.get(adapter_name, None)
         if hasattr(p, 'ip_adapter_name'):
             adapter = p.ip_adapter_name
         if hasattr(p, 'ip_adapter_scale'):
@@ -60,7 +62,7 @@ class Script(scripts.Script):
         if adapter is None:
             return
         # init code
-        global loaded, checkpoint, image_encoder, image_encoder_type # pylint: disable=global-statement
+        global loaded, checkpoint, image_encoder, image_encoder_type, image_encoder_name # pylint: disable=global-statement
         if shared.sd_model is None:
             return
         if shared.backend != shared.Backend.DIFFUSERS:
@@ -80,25 +82,39 @@ class Script(scripts.Script):
         if not hasattr(shared.sd_model, 'load_ip_adapter'):
             shared.log.error(f'IP adapter: pipeline not supported: {shared.sd_model.__class__.__name__}')
             return
-        if getattr(shared.sd_model, 'image_encoder', None) is None:
-            if shared.sd_model_type == 'sd':
-                subfolder = 'models/image_encoder'
-            elif shared.sd_model_type == 'sdxl':
-                subfolder = 'sdxl_models/image_encoder'
-            else:
-                shared.log.error(f'IP adapter: unsupported model type: {shared.sd_model_type}')
-                return
-            if image_encoder is None or image_encoder_type != shared.sd_model_type or checkpoint != shared.opts.sd_model_checkpoint:
+
+        # which clip to use
+        if 'ViT' not in adapter_name:
+            clip_repo = base_repo
+            subfolder = 'models/image_encoder' if shared.sd_model_type == 'sd' else 'sdxl_models/image_encoder' # defaults per model
+        elif 'ViT-H' in adapter_name:
+            clip_repo = base_repo
+            subfolder = 'models/image_encoder' # this is vit-h
+        elif 'ViT-G' in adapter_name:
+            clip_repo = base_repo
+            subfolder = 'sdxl_models/image_encoder' # this is vit-g
+        else:
+            shared.log.error(f'IP adapter: unknown model type: {adapter_name}')
+            return
+
+        # load image encoder used by ip adapter
+        if getattr(shared.sd_model, 'image_encoder', None) is None or image_encoder_name != clip_repo + '/' + subfolder:
+            if image_encoder is None or image_encoder_type != shared.sd_model_type or checkpoint != shared.opts.sd_model_checkpoint or image_encoder_name != clip_repo + '/' + subfolder:
+                if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
+                    shared.log.error(f'IP adapter: unsupported model type: {shared.sd_model_type}')
+                    return
                 try:
                     from transformers import CLIPVisionModelWithProjection
-                    image_encoder = CLIPVisionModelWithProjection.from_pretrained("h94/IP-Adapter", subfolder=subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True).to(devices.device)
+                    shared.log.debug(f'IP adapter: load image encoder: {clip_repo}/{subfolder}')
+                    image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True).to(devices.device)
                     image_encoder_type = shared.sd_model_type
+                    image_encoder_name = clip_repo + '/' + subfolder
                 except Exception as e:
                     shared.log.error(f'IP adapter: failed to load image encoder: {e}')
                     return
 
         # main code
-        subfolder = 'models' if 'sd15' in adapter else 'sdxl_models'
+        # subfolder = 'models' if 'sd15' in adapter else 'sdxl_models'
         if adapter != loaded or getattr(shared.sd_model.unet.config, 'encoder_hid_dim_type', None) is None or checkpoint != shared.opts.sd_model_checkpoint:
             t0 = time.time()
             if loaded is not None:
@@ -107,7 +123,8 @@ class Script(scripts.Script):
             else:
                 shared.log.debug('IP adapter: load attention processor')
             shared.sd_model.image_encoder = image_encoder
-            shared.sd_model.load_ip_adapter("h94/IP-Adapter", subfolder=subfolder, weight_name=f'{adapter}.safetensors')
+            subfolder = 'models' if shared.sd_model_type == 'sd' else 'sdxl_models'
+            shared.sd_model.load_ip_adapter(base_repo, subfolder=subfolder, weight_name=adapter)
             t1 = time.time()
             shared.log.info(f'IP adapter load: adapter="{adapter}" scale={scale} image={image} time={t1-t0:.2f}')
             loaded = adapter