add slg

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -16,7 +16,7 @@ First, a massive update to docs including new UI top-level **info** tab with acc
 
 **Workflow Improvements**:
 - Native Docker support
-- SD3x & Flux.1: more ControlNets, all-in-one-safetensors, DPM samplers, etc.
+- SD3x & Flux.1: more ControlNets, all-in-one-safetensors, DPM samplers, skip-layer-guidance, etc.
 - XYZ grid: benchmarking, video creation, etc.
 - Enhanced prompt parsing
 - UI improvements
@@ -66,15 +66,18 @@ And quite a few more improvements and fixes since the last update - for full det
       `<MIXED_CAPTION>`, `<MIXED_CAPTION_PLUS>` detailed caption and tags with optional analyze  
 
 - Model improvements:  
-  - SD3: ControlNets:  
+  - SD35: **ControlNets**:  
     - *InstantX Canny, Pose, Depth, Tile*  
     - *Alimama Inpainting, SoftEdge*  
     - *note*: that just like with FLUX.1 or any large model, ControlNet are also large and can push your system over the limit  
       e.g. SD3 controlnets vary from 1GB to over 4GB in size  
-  - SD3: all-in-one safetensors  
+  - SD35: **All-in-one** safetensors  
     - *examples*: [large](https://civitai.com/models/882666/sd35-large-google-flan?modelVersionId=1003031), [medium](https://civitai.com/models/900327)  
     - *note*: enable *bnb* on-the-fly quantization for even bigger gains  
-  - FlowMatch samplers:
+  - SD35: **skip-layer-guidance**  
+    - enable in *scripts -> slg*
+    - allows for granular strength/start/stop control of guidance for each layer of the model  
+  - **FlowMatch samplers**:
     - Applicable to SD 3.x and Flux.1 models
     - Complete family: *DPM2, DPM2a, DPM2++, DPM2++ 2M, DPM2++ 2S, DPM2++ SDE, DPM2++ 2M SDE, DPM2++ 3M SDE*
   - [NoobAI XL ControlNets](https://huggingface.co/collections/Eugeoter/controlnext-673161eae023f413e0432799), thanks @lbeltrame

installer.py

@@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None):
 def check_diffusers():
     if args.skip_all or args.skip_requirements:
         return
-    sha = '345907f32de71c8ca67f3d9d00e37127192da543'
+    sha = '99c0483b67427de467f11aa35d54678fd36a7ea2'
     pkg = pkg_resources.working_set.by_key.get('diffusers', None)
     minor = int(pkg.version.split('.')[1] if pkg is not None else 0)
     cur = opts.get('diffusers_version', '') if minor > 0 else ''

modules/ipadapter.py

@@ -35,8 +35,15 @@ ADAPTERS_SDXL = {
     'Plus Face ViT-H SDXL': { 'name': 'ip-adapter-plus-face_sdxl_vit-h.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'sdxl_models' },
     'Ostris Composition ViT-H SDXL': { 'name': 'ip_plus_composition_sdxl.safetensors', 'repo': 'ostris/ip-composition-adapter', 'subfolder': '' },
 }
-ADAPTERS = { **ADAPTERS_SD15, **ADAPTERS_SDXL }
-ADAPTERS_ALL = { **ADAPTERS_SD15, **ADAPTERS_SDXL }
+ADAPTERS_SD3 = {
+    'InstantX Large': { 'name': 'ip-adapter.bin', 'repo': 'InstantX/SD3.5-Large-IP-Adapter' },
+}
+ADAPTERS_F1 = {
+    'XLabs AI v1': { 'name': 'ip_adapter.safetensors', 'repo': 'XLabs-AI/flux-ip-adapter' },
+    'XLabs AI v2': { 'name': 'ip_adapter.safetensors', 'repo': 'XLabs-AI/flux-ip-adapter-v2' },
+}
+ADAPTERS = { **ADAPTERS_SD15, **ADAPTERS_SDXL, **ADAPTERS_SD3, **ADAPTERS_F1 }
+ADAPTERS_ALL = { **ADAPTERS_SD15, **ADAPTERS_SDXL, **ADAPTERS_SD3, **ADAPTERS_F1 }
 
 
 def get_adapters():
@@ -45,6 +52,10 @@ def get_adapters():
         ADAPTERS = ADAPTERS_SD15
     elif shared.sd_model_type == 'sdxl':
         ADAPTERS = ADAPTERS_SDXL
+    elif shared.sd_model_type == 'sd3':
+        ADAPTERS = ADAPTERS_SD3
+    elif shared.sd_model_type == 'f1':
+        ADAPTERS = ADAPTERS_F1
     else:
         ADAPTERS = ADAPTERS_NONE
     return list(ADAPTERS)
@@ -55,7 +66,7 @@ def get_images(input_images):
     if input_images is None or len(input_images) == 0:
         shared.log.error('IP adapter: no init images')
         return None
-    if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
+    if shared.sd_model_type not in ['sd', 'sdxl', 'sd3', 'f1']:
         shared.log.error('IP adapter: base model not supported')
         return None
     if isinstance(input_images, str):
@@ -147,7 +158,7 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt
         if hasattr(p, 'ip_adapter_images'):
             del p.ip_adapter_images
         return False
-    if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
+    if shared.sd_model_type not in ['sd', 'sdxl', 'sd3', 'f1']:
         shared.log.error(f'IP adapter: model={shared.sd_model_type} class={pipe.__class__.__name__} not supported')
         return False
     if hasattr(p, 'ip_adapter_scales'):
@@ -172,6 +183,9 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt
         for i in range(len(adapter_masks)):
             adapter_masks[i] = mask_processor.preprocess(adapter_masks[i], height=p.height, width=p.width)
         adapter_masks = mask_processor.preprocess(adapter_masks, height=p.height, width=p.width)
+    if adapter_images is None:
+        shared.log.error('IP adapter: no image provided')
+        return False
     if len(adapters) < len(adapter_images):
         adapter_images = adapter_images[:len(adapters)]
     if len(adapters) < len(adapter_masks):
@@ -212,13 +226,24 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt
                 clip_subfolder = 'models/image_encoder'
             else:
                 clip_subfolder = 'sdxl_models/image_encoder'
-        elif 'ViT-H' in adapter_name:
+        if 'ViT-H' in adapter_name:
             clip_subfolder = 'models/image_encoder' # this is vit-h
         elif 'ViT-G' in adapter_name:
             clip_subfolder = 'sdxl_models/image_encoder' # this is vit-g
         else:
-            shared.log.error(f'IP adapter: unknown model type: {adapter_name}')
-            return False
+            if shared.sd_model_type == 'sd':
+                clip_subfolder = 'models/image_encoder'
+            elif shared.sd_model_type == 'sdxl':
+                clip_subfolder = 'sdxl_models/image_encoder'
+            elif shared.sd_model_type == 'sd3':
+                shared.log.error(f'IP adapter: adapter={adapter_name} type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__}: unsupported base model')
+                return False
+            elif shared.sd_model_type == 'f1':
+                shared.log.error(f'IP adapter: adapter={adapter_name} type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__}: unsupported base model')
+                return False
+            else:
+                shared.log.error(f'IP adapter: unknown model type: {adapter_name}')
+                return False
 
     # load feature extractor used by ip adapter
     if pipe.feature_extractor is None:

scripts/skip_layer_guidance.py

@@ -0,0 +1,71 @@
+import sys
+import gradio as gr
+from modules import scripts, processing, shared
+
+
+registered = False
+
+
+class Script(scripts.Script):
+    def __init__(self):
+        super().__init__()
+        self.register()
+
+    def title(self):
+        return 'SLG: Skip Layer Guidance'
+
+    def show(self, is_img2img):
+        return shared.native
+
+    # return signature is array of gradio components
+    def ui(self, _is_img2img):
+        with gr.Row():
+            layers = gr.Textbox(label='Skip guidance layers', value='7,8,9')
+        with gr.Row():
+            scale = gr.Slider(label='Guidance strength', minimum=0.0, maximum=1.0, step=0.01, value=1.0)
+        with gr.Row():
+            start = gr.Slider(label='Guidance start', minimum=0.0, maximum=1.0, step=0.01, value=0.01)
+            stop = gr.Slider(label='Guidance stop', minimum=0.0, maximum=1.0, step=0.01, value=0.2)
+        return [layers, scale, start, stop]
+
+    def register(self): # register xyz grid elements
+        global registered # pylint: disable=global-statement
+        if registered:
+            return
+        registered = True
+        def apply_task_args(field):
+            def fun(p, x, xs): # pylint: disable=unused-argument
+                try:
+                    val = str(x).replace('"', '')
+                    val = [int(layer.strip()) for layer in val.split(',')]
+                except Exception:
+                    return
+                if len(val) > 0:
+                    shared.log.debug(f'SLG: {field}={val}')
+                    p.task_args[field] = val
+            return fun
+
+        xyz_classes = [v for k, v in sys.modules.items() if 'xyz_grid_classes' in k][0]
+        options = [
+            xyz_classes.AxisOption("[SLG] Layers", str, apply_task_args("skip_guidance_layers")),
+        ]
+        for option in options:
+            if option not in xyz_classes.axis_options:
+                xyz_classes.axis_options.append(option)
+
+
+    def run(self, p: processing.StableDiffusionProcessing, layers: str = '', scale: float = 1.0, start: float = 1.0, stop: float = 1.0): # pylint: disable=arguments-differ, unused-argument
+        if shared.sd_model_type != 'sd3':
+            return
+        p.task_args['skip_layer_guidance_scale'] = float(scale)
+        p.task_args['skip_layer_guidance_start'] = float(start)
+        p.task_args['skip_layer_guidance_stop'] = float(stop)
+        parsed = []
+        try:
+            parsed = [int(layer.strip()) for layer in layers.split(',')]
+        except Exception:
+            return
+        if len(parsed) == 0:
+            return
+        p.task_args['skip_guidance_layers'] = parsed
+        shared.log.info(f'SLG: layers={parsed} scale={scale} start={start} stop={stop}')