refactor control processing

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -1,8 +1,8 @@
 # Change Log for SD.Next
 
-## Update for 2025-07-15
+## Update for 2025-07-16
 
-### Highlights for 2025-07-15
+### Highlights for 2025-07-16
 
 In this release we finally break with legacy with the removal of the original [A1111](https://github.com/AUTOMATIC1111/stable-diffusion-webui/) codebase which has not been maintained for a while now  
 This plus major cleanup of codebase and external dependencies resulted in ~53k LoC (*lines-of-code*) reduction and spread over [~720 files](https://github.com/vladmandic/sdnext/pull/4017)!  
@@ -24,7 +24,7 @@ Although upgrades and existing installations are tested and should work fine!
 
 [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867)
 
-### Details for 2025-07-15
+### Details for 2025-07-16
 
 - **License**  
   - SD.Next [license](https://github.com/vladmandic/sdnext/blob/dev/LICENSE.txt) switched from **aGPL-v3.0** to **Apache-v2.0**  
@@ -113,6 +113,8 @@ Although upgrades and existing installations are tested and should work fine!
   - fix incorrect reporting of deleted and modified files  
   - fix SD3.x loader and TAESD preview  
   - fix xyz with control enabled  
+  - fix control order of image save operations  
+  - cleanup control infotext  
   - allow upscaling with models that have implicit VAE processing  
   - sdnq use inference context during quantization  
   - framepack improve offloading  
@@ -139,7 +141,8 @@ Although upgrades and existing installations are tested and should work fine!
   - remove legacy lora support: `/extensions-builtin/Lora`
   - remove legacy clip/blip interrogate module
   - remove modern-ui remove `only-original` vs `only-diffusers` code paths  
-  - split monolithic `shared.py`
+  - refactor control processing and separate preprocessing and image save ops  
+  - split monolithic `shared.py`  
   - cleanup `/modules`: move pipeline loaders to `/pipelines` root  
   - cleanup `/modules`: move code folders used by pipelines to `/pipelines/<pipeline>` folder  
   - cleanup `/modules`: move code folders used by scripts to `/scripts/<script>` folder  

modules/control/processor.py

@@ -0,0 +1,247 @@
+import os
+import time
+import hashlib
+import numpy as np
+from PIL import Image
+from modules.processing_class import StableDiffusionProcessingControl
+from modules import shared, images, masking, sd_models
+from modules.timer import process as process_timer
+from modules.control import util
+
+
+debug = os.environ.get('SD_CONTROL_DEBUG', None) is not None
+debug_log = shared.log.trace if debug else lambda *args, **kwargs: None
+processors = [
+    'None',
+    'OpenPose',
+    'DWPose',
+    'MediaPipe Face',
+    'Canny',
+    'Edge',
+    'LineArt Realistic',
+    'LineArt Anime',
+    'HED',
+    'PidiNet',
+    'Midas Depth Hybrid',
+    'Leres Depth',
+    'Zoe Depth',
+    'Marigold Depth',
+    'Normal Bae',
+    'SegmentAnything',
+    'MLSD',
+    'Shuffle',
+    'DPT Depth Hybrid',
+    'GLPN Depth',
+    'Depth Anything',
+]
+
+
+def preprocess_image(
+        p:StableDiffusionProcessingControl,
+        pipe,
+        input_image:Image.Image,
+        init_image:Image.Image,
+        input_mask:Image.Image,
+        input_type:str,
+        unit_type:str,
+        active_process:list,
+        active_model:list,
+        selected_models:list,
+        has_models:bool,
+    ):
+    t0 = time.time()
+
+    # run resize before
+    if p.resize_mode_before != 0 and p.resize_name_before != 'None':
+        if p.selected_scale_tab_before == 1 and input_image is not None:
+            p.width_before, p.height_before = int(input_image.width * p.scale_by_before), int(input_image.height * p.scale_by_before)
+        if input_image is not None:
+            debug_log(f'Control resize: op=before image={input_image} width={p.width_before} height={p.height_before} mode={p.resize_mode_before} name={p.resize_name_before} context="{p.resize_context_before}"')
+            p.init_img_hash = getattr(p, 'init_img_hash', hashlib.sha256(input_image.tobytes()).hexdigest()[0:8]) # pylint: disable=attribute-defined-outside-init
+            p.init_img_width = getattr(p, 'init_img_width', input_image.width) # pylint: disable=attribute-defined-outside-init
+            p.init_img_height = getattr(p, 'init_img_height', input_image.height) # pylint: disable=attribute-defined-outside-init
+            input_image = images.resize_image(p.resize_mode_before, input_image, p.width_before, p.height_before, p.resize_name_before, context=p.resize_context_before)
+    if input_image is not None and init_image is not None and init_image.size != input_image.size:
+        debug_log(f'Control resize init: image={init_image} target={input_image}')
+        init_image = images.resize_image(resize_mode=1, im=init_image, width=input_image.width, height=input_image.height)
+    if input_image is not None and p.override is not None and p.override.size != input_image.size:
+        debug_log(f'Control resize override: image={p.override} target={input_image}')
+        p.override = images.resize_image(resize_mode=1, im=p.override, width=input_image.width, height=input_image.height)
+    if input_image is not None:
+        p.width = input_image.width
+        p.height = input_image.height
+        debug_log(f'Control: input image={input_image}')
+
+    # run masking
+    if input_mask is not None:
+        p.extra_generation_params["Mask only"] = masking.opts.mask_only if masking.opts.mask_only else None
+        p.extra_generation_params["Mask auto"] = masking.opts.auto_mask if masking.opts.auto_mask != 'None' else None
+        p.extra_generation_params["Mask invert"] = masking.opts.invert if masking.opts.invert else None
+        p.extra_generation_params["Mask blur"] = masking.opts.mask_blur if masking.opts.mask_blur > 0 else None
+        p.extra_generation_params["Mask erode"] = masking.opts.mask_erode if masking.opts.mask_erode > 0 else None
+        p.extra_generation_params["Mask dilate"] = masking.opts.mask_dilate if masking.opts.mask_dilate > 0 else None
+        p.extra_generation_params["Mask model"] = masking.opts.model if masking.opts.model is not None else None
+        masked_image = masking.run_mask(input_image=input_image, input_mask=input_mask, return_type='Masked', invert=p.inpainting_mask_invert==1) if input_mask is not None else input_image
+    else:
+        masked_image = input_image
+
+    # resize mask
+    if input_mask is not None and p.resize_mode_mask != 0 and p.resize_name_mask != 'None':
+        if p.selected_scale_tab_mask == 1:
+            p.width_mask, p.height_mask = int(input_image.width * p.scale_by_mask), int(input_image.height * p.scale_by_mask)
+        p.width, p.height = p.width_mask, p.height_mask
+        debug_log(f'Control resize: op=mask image={input_mask} width={p.width_mask} height={p.height_mask} mode={p.resize_mode_mask} name={p.resize_name_mask} context="{p.resize_context_mask}"')
+
+    # run image processors
+    processed_images = []
+    for i, process in enumerate(active_process): # list[image]
+        debug_log(f'Control: i={i+1} process="{process.processor_id}" input={masked_image} override={process.override}')
+        processed_image = process(
+            image_input=masked_image,
+            mode='RGB',
+            resize_mode=p.resize_mode_before,
+            resize_name=p.resize_name_before,
+            scale_tab=p.selected_scale_tab_before,
+            scale_by=p.scale_by_before,
+        )
+        if processed_image is not None:
+            processed_images.append(processed_image)
+        if shared.opts.control_unload_processor and process.processor_id is not None:
+            processors.config[process.processor_id]['dirty'] = True # to force reload
+            process.model = None
+
+    # blend processed images
+    debug_log(f'Control processed: {len(processed_images)}')
+    if len(processed_images) > 0:
+        try:
+            if len(p.extra_generation_params["Control process"]) == 0:
+                p.extra_generation_params["Control process"] = None
+            else:
+                p.extra_generation_params["Control process"] = ';'.join([p.processor_id for p in active_process if p.processor_id is not None])
+        except Exception:
+            pass
+        if any(img is None for img in processed_images):
+            shared.log.error('Control: one or more processed images are None')
+            processed_images = [img for img in processed_images if img is not None]
+        if len(processed_images) > 1 and len(active_process) != len(active_model):
+            processed_image = [np.array(i) for i in processed_images]
+            processed_image = util.blend(processed_image) # blend all processed images into one
+            processed_image = Image.fromarray(processed_image)
+            blended_image = processed_image
+        elif len(processed_images) == 1:
+            processed_image = processed_images
+            blended_image = processed_image[0]
+        else:
+            blended_image = [np.array(i) for i in processed_images]
+            blended_image = util.blend(blended_image) # blend all processed images into one
+            blended_image = Image.fromarray(blended_image)
+        if isinstance(selected_models, list) and len(processed_images) == len(selected_models) and len(processed_images) > 0:
+            debug_log(f'Control: inputs match: input={len(processed_images)} models={len(selected_models)}')
+            p.init_images = processed_images
+        elif isinstance(selected_models, list) and len(processed_images) != len(selected_models):
+            shared.log.error(f'Control: number of inputs does not match: input={len(processed_images)} models={len(selected_models)}')
+        elif selected_models is not None:
+            p.init_images = processed_image
+    else:
+        debug_log('Control processed: using input direct')
+        processed_image = input_image
+
+    # conditional assignment
+    possible = sd_models.get_call(pipe).keys()
+    if unit_type == 'reference' and has_models:
+        p.ref_image = p.override or input_image
+        p.task_args.pop('image', None)
+        p.task_args['ref_image'] = p.ref_image
+        debug_log(f'Control: process=None image={p.ref_image}')
+        if p.ref_image is None:
+            shared.log.error('Control: reference mode without image')
+    elif unit_type == 'controlnet' and has_models:
+        if input_type == 0: # Control only
+            if 'control_image' in possible:
+                p.task_args['control_image'] = [p.init_images] if isinstance(p.init_images, Image.Image) else p.init_images
+            elif 'image' in possible:
+                p.task_args['image'] = [p.init_images] if isinstance(p.init_images, Image.Image) else p.init_images
+            if 'control_mode' in possible:
+                p.task_args['control_mode'] = getattr(p, 'control_mode', None)
+            if 'strength' in possible:
+                p.task_args['strength'] = p.denoising_strength
+            p.init_images = None
+        elif input_type == 1: # Init image same as control
+            p.init_images = [p.override or input_image] * max(1, len(active_model))
+            if 'inpaint_image' in possible: # flex
+                p.task_args['inpaint_image'] = p.init_images[0] if isinstance(p.init_images, list) else p.init_images
+                p.task_args['inpaint_mask'] = Image.new('L', p.task_args['inpaint_image'].size, int(p.denoising_strength * 255))
+                p.task_args['control_image'] = p.init_images[0] if isinstance(p.init_images, list) else p.init_images
+                p.task_args['width'] = p.width
+                p.task_args['height'] = p.height
+            elif 'control_image' in possible:
+                p.task_args['control_image'] = p.init_images # switch image and control_image
+            if 'control_mode' in possible:
+                p.task_args['control_mode'] = getattr(p, 'control_mode', None)
+            if 'strength' in possible:
+                p.task_args['strength'] = p.denoising_strength
+        elif input_type == 2: # Separate init image
+            if init_image is None:
+                shared.log.warning('Control: separate init image not provided')
+                init_image = input_image
+            if 'inpaint_image' in possible: # flex
+                p.task_args['inpaint_image'] = p.init_images[0] if isinstance(p.init_images, list) else p.init_images
+                p.task_args['inpaint_mask'] = Image.new('L', p.task_args['inpaint_image'].size, int(p.denoising_strength * 255))
+                p.task_args['control_image'] = p.init_images[0] if isinstance(p.init_images, list) else p.init_images
+                p.task_args['width'] = p.width
+                p.task_args['height'] = p.height
+            elif 'control_image' in possible:
+                p.task_args['control_image'] = p.init_images # switch image and control_image
+            if 'control_mode' in possible:
+                p.task_args['control_mode'] = getattr(p, 'control_mode', None)
+            if 'strength' in possible:
+                p.task_args['strength'] = p.denoising_strength
+            p.init_images = [init_image] * len(active_model)
+        if hasattr(shared.sd_model, 'controlnet') and hasattr(p.task_args, 'control_image') and len(p.task_args['control_image']) > 1 and (shared.sd_model.__class__.__name__ == 'StableDiffusionXLControlNetUnionPipeline'): # special case for controlnet-union
+            p.task_args['control_image'] = [[x] for x in p.task_args['control_image']]
+            p.task_args['control_mode'] = [[x] for x in p.task_args['control_mode']]
+
+    # determine txt2img, img2img, inpaint pipeline
+    if unit_type == 'reference' and has_models: # special case
+        p.is_control = True
+        shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
+    elif not has_models: # run in txt2img/img2img/inpaint mode
+        if input_mask is not None:
+            p.task_args['strength'] = p.denoising_strength
+            p.image_mask = input_mask
+            p.init_images = input_image if isinstance(input_image, list) else [input_image]
+            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.INPAINTING)
+        elif processed_image is not None:
+            p.init_images = processed_image if isinstance(processed_image, list) else [processed_image]
+            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
+        else:
+            p.init_hr(p.scale_by, p.resize_name, force=True)
+            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
+    elif has_models: # actual control
+        p.is_control = True
+        if input_mask is not None:
+            p.task_args['strength'] = p.denoising_strength
+            p.image_mask = input_mask
+            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.INPAINTING) # only controlnet supports inpaint
+        if hasattr(p, 'init_images') and p.init_images is not None:
+            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) # only controlnet supports img2img
+        else:
+            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
+            if hasattr(p, 'init_images') and p.init_images is not None and 'image' in possible:
+                p.task_args['image'] = p.init_images # need to set explicitly for txt2img
+                p.init_images = None
+        if unit_type == 'lite':
+            if input_type == 0:
+                shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
+                shared.sd_model.no_task_switch = True
+            elif input_type == 1:
+                p.init_images = [input_image]
+            elif input_type == 2:
+                if init_image is None:
+                    shared.log.warning('Control: separate init image not provided')
+                    init_image = input_image
+                p.init_images = [init_image]
+
+    t1 = time.time()
+    process_timer.add('proc', t1-t0)
+    return processed_image

modules/control/processors_list.py

@@ -1,23 +0,0 @@
-processors = [
-    'None',
-    'OpenPose',
-    'DWPose',
-    'MediaPipe Face',
-    'Canny',
-    'Edge',
-    'LineArt Realistic',
-    'LineArt Anime',
-    'HED',
-    'PidiNet',
-    'Midas Depth Hybrid',
-    'Leres Depth',
-    'Zoe Depth',
-    'Marigold Depth',
-    'Normal Bae',
-    'SegmentAnything',
-    'MLSD',
-    'Shuffle',
-    'DPT Depth Hybrid',
-    'GLPN Depth',
-    'Depth Anything',
-]

modules/control/run.py

@@ -1,8 +1,7 @@
 import os
-import time
+import sys
 from typing import List, Union
 import cv2
-import numpy as np
 from PIL import Image
 from modules.control import util # helper functions
 from modules.control import unit # control units
@@ -15,9 +14,9 @@ from modules.control.units import t2iadapter # TencentARC T2I-Adapter
 from modules.control.units import reference # ControlNet-Reference
 from modules import devices, shared, errors, processing, images, sd_models, scripts_manager, masking
 from modules.processing_class import StableDiffusionProcessingControl
+from modules.processing_info import create_infotext
 from modules.ui_common import infotext_to_html
 from modules.api import script
-from modules.timer import process as process_timer
 
 
 debug = os.environ.get('SD_CONTROL_DEBUG', None) is not None
@@ -31,10 +30,11 @@ unified_models = ['Flex2Pipeline'] # models that have controlnet builtin
 
 def restore_pipeline():
     global pipe, instance # pylint: disable=global-statement
+    fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
     if instance is not None and hasattr(instance, 'restore'):
         instance.restore()
     if (original_pipeline is not None) and (original_pipeline.__class__.__name__ != shared.sd_model.__class__.__name__):
-        debug_log(f'Control restored pipeline: class={shared.sd_model.__class__.__name__} to={original_pipeline.__class__.__name__}')
+        debug_log(f'Control restored pipeline: class={shared.sd_model.__class__.__name__} to={original_pipeline.__class__.__name__} fn={fn}')
         shared.sd_model = original_pipeline
     pipe = None
     instance = None
@@ -269,6 +269,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
         ):
     global pipe, original_pipeline # pylint: disable=global-statement
 
+    unit.current = units
     debug_log(f'Control: type={unit_type} input={inputs} init={inits} type={input_type}')
     init_units(units)
     if inputs is None or (type(inputs) is list and len(inputs) == 0):
@@ -297,6 +298,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
         subseed_strength = subseed_strength,
         seed_resize_from_h = seed_resize_from_h,
         seed_resize_from_w = seed_resize_from_w,
+        denoising_strength = denoising_strength,
         # advanced
         cfg_scale = cfg_scale,
         cfg_end = cfg_end,
@@ -308,18 +310,53 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
         vae_type = vae_type,
         tiling = tiling,
         hidiffusion = hidiffusion,
+        # resize
+        width = width_before,
+        height = height_before,
+        width_before = width_before,
+        width_after = width_after,
+        width_mask = width_mask,
+        height_before = height_before,
+        height_after = height_after,
+        height_mask = height_mask,
+        resize_name_before = resize_name_before,
+        resize_name_after = resize_name_after,
+        resize_name_mask = resize_name_mask,
+        resize_mode_before = resize_mode_before if resize_name_before != 'None' and inputs is not None and len(inputs) > 0 else 0,
+        resize_mode_after = resize_mode_after if resize_name_after != 'None' else 0,
+        resize_mode_mask = resize_mode_mask if resize_name_mask != 'None' else 0,
+        resize_context_before = resize_context_before,
+        resize_context_after = resize_context_after,
+        resize_context_mask = resize_context_mask,
+        selected_scale_tab_before = selected_scale_tab_before,
+        selected_scale_tab_after = selected_scale_tab_after,
+        selected_scale_tab_mask = selected_scale_tab_mask,
+        scale_by_before = scale_by_before,
+        scale_by_after = scale_by_after,
+        scale_by_mask = scale_by_mask,
+        # hires
+        enable_hr = enable_hr,
+        hr_sampler_name = processing.get_sampler_name(hr_sampler_index),
+        hr_denoising_strength = hr_denoising_strength,
+        hr_resize_mode = hr_resize_mode if enable_hr else 0,
+        hr_resize_context = hr_resize_context if enable_hr else 'None',
+        hr_upscaler = hr_upscaler if enable_hr else None,
+        hr_force = hr_force,
+        hr_second_pass_steps = hr_second_pass_steps if enable_hr else 0,
+        hr_scale = hr_scale if enable_hr else 1.0,
+        hr_resize_x = hr_resize_x if enable_hr else 0,
+        hr_resize_y = hr_resize_y if enable_hr else 0,
+        # refiner
+        refiner_steps = refiner_steps,
+        refiner_start = refiner_start,
+        refiner_prompt = refiner_prompt,
+        refiner_negative = refiner_negative,
         # detailer
         detailer_enabled = detailer_enabled,
         detailer_prompt = detailer_prompt,
         detailer_negative = detailer_negative,
         detailer_steps = detailer_steps,
         detailer_strength = detailer_strength,
-        # resize
-        resize_mode = resize_mode_before if resize_name_before != 'None' else 0,
-        resize_name = resize_name_before,
-        scale_by = scale_by_before,
-        selected_scale_tab = selected_scale_tab_before,
-        denoising_strength = denoising_strength,
         # inpaint
         inpaint_full_res = masking.opts.mask_only,
         inpainting_mask_invert = 1 if masking.opts.invert else 0,
@@ -332,69 +369,23 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
     )
     p.state = state
     p.is_tile = False
-    # processing.process_init(p)
-    resize_mode_before = resize_mode_before if resize_name_before != 'None' and inputs is not None and len(inputs) > 0 else 0
 
     # TODO modernui: monkey-patch for missing tabs.select event
-    if selected_scale_tab_before == 0 and resize_name_before != 'None' and scale_by_before != 1 and inputs is not None and len(inputs) > 0:
+    if p.selected_scale_tab_before == 0 and p.resize_name_before != 'None' and p.scale_by_before != 1 and inputs is not None and len(inputs) > 0:
         shared.log.debug('Control: override resize mode=before')
-        selected_scale_tab_before = 1
-    if selected_scale_tab_after == 0 and resize_name_after != 'None' and scale_by_after != 1:
+        p.selected_scale_tab_before = 1
+    if p.selected_scale_tab_after == 0 and p.resize_name_after != 'None' and p.scale_by_after != 1:
         shared.log.debug('Control: override resize mode=after')
-        selected_scale_tab_after = 1
-    if selected_scale_tab_mask == 0 and resize_name_mask != 'None' and scale_by_mask != 1:
+        p.selected_scale_tab_after = 1
+    if p.selected_scale_tab_mask == 0 and p.resize_name_mask != 'None' and p.scale_by_mask != 1:
         shared.log.debug('Control: override resize mode=mask')
-        selected_scale_tab_mask = 1
-
-    # set control sizing
-    if resize_mode_before != 0 or inputs is None or inputs == [None]:
-        p.width, p.height = width_before, height_before # pylint: disable=attribute-defined-outside-init
-        p.width_before = width_before
-        p.height_before = height_before
-        if resize_name_before != 'None':
-            p.resize_mode_before = resize_mode_before
-            p.resize_name_before = resize_name_before
-        p.scale_by_before = scale_by_before
-        p.selected_scale_tab_before = selected_scale_tab_before
-    else:
-        del p.width
-        del p.height
-    if resize_name_after != 'None':
-        p.resize_mode_after = resize_mode_after
-        p.resize_name_after = resize_name_after
-        p.width_after = width_after
-        p.height_after = height_after
-        p.scale_by_after = scale_by_after
-        p.selected_scale_tab_after = selected_scale_tab_after
-    if resize_name_mask != 'None':
-        p.resize_mode_mask = resize_mode_mask
-        p.resize_name_mask = resize_name_mask
-        p.width_mask = width_mask
-        p.height_mask = height_mask
-        p.scale_by_mask = scale_by_mask
-        p.selected_scale_tab_mask = selected_scale_tab_mask
+        p.selected_scale_tab_mask = 1
 
     # hires/refine defined outside of main init
-    p.enable_hr = enable_hr
-    p.hr_sampler_name = processing.get_sampler_name(hr_sampler_index)
-    p.hr_denoising_strength = hr_denoising_strength
-    p.hr_resize_mode = hr_resize_mode
-    p.hr_resize_context = hr_resize_context
-    p.hr_upscaler = hr_upscaler
-    p.hr_force = hr_force
-    p.hr_second_pass_steps = hr_second_pass_steps
-    p.hr_scale = hr_scale
-    p.hr_resize_x = hr_resize_x
-    p.hr_resize_y = hr_resize_y
-    p.refiner_steps = refiner_steps
-    p.refiner_start = refiner_start
-    p.refiner_prompt = refiner_prompt
-    p.refiner_negative = refiner_negative
     if p.enable_hr and (p.hr_resize_x == 0 or p.hr_resize_y == 0):
-        p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(width_before * p.hr_scale / 8), 8 * int(height_before * p.hr_scale / 8)
+        p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.width_before * p.hr_scale / 8), 8 * int(p.height_before * p.hr_scale / 8)
     elif p.enable_hr and (p.hr_upscale_to_x == 0 or p.hr_upscale_to_y == 0):
-        p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.hr_resize_x / 8), 8 * int(hr_resize_y / 8)
-
+        p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.hr_resize_x / 8), 8 * int(p.hr_resize_y / 8)
 
     global p_extra_args # pylint: disable=global-statement
     for k, v in p_extra_args.items():
@@ -406,8 +397,6 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
         return [], '', '', 'Error: model not loaded'
 
     unit_type = unit_type.strip().lower() if unit_type is not None else ''
-    t0 = time.time()
-
     active_process, active_model, active_strength, active_start, active_end = check_active(p, unit_type, units)
     has_models, selected_models, control_conditioning, control_guidance_start, control_guidance_end = check_enabled(p, unit_type, units, active_model, active_strength, active_start, active_end)
 
@@ -421,7 +410,6 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
 
     pipe = set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, control_conditioning, control_guidance_start, control_guidance_end, inits)
     debug_log(f'Control pipeline: class={pipe.__class__.__name__} args={vars(p)}')
-    t1, t2, t3 = time.time(), 0, 0
     status = True
     frame = None
     video = None
@@ -517,216 +505,23 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
                         continue
                     index += 1
 
-                    # resize before
-                    if resize_mode_before != 0 and resize_name_before != 'None':
-                        if selected_scale_tab_before == 1 and input_image is not None:
-                            width_before, height_before = int(input_image.width * scale_by_before), int(input_image.height * scale_by_before)
-                        if input_image is not None:
-                            p.extra_generation_params["Control resize"] = f'{resize_name_before}'
-                            debug_log(f'Control resize: op=before image={input_image} width={width_before} height={height_before} mode={resize_mode_before} name={resize_name_before} context="{resize_context_before}"')
-                            input_image = images.resize_image(resize_mode_before, input_image, width_before, height_before, resize_name_before, context=resize_context_before)
-                    if input_image is not None and init_image is not None and init_image.size != input_image.size:
-                        debug_log(f'Control resize init: image={init_image} target={input_image}')
-                        init_image = images.resize_image(resize_mode=1, im=init_image, width=input_image.width, height=input_image.height)
-                    if input_image is not None and p.override is not None and p.override.size != input_image.size:
-                        debug_log(f'Control resize override: image={p.override} target={input_image}')
-                        p.override = images.resize_image(resize_mode=1, im=p.override, width=input_image.width, height=input_image.height)
-                    if input_image is not None:
-                        p.width = input_image.width
-                        p.height = input_image.height
-                        debug_log(f'Control: input image={input_image}')
-
-                    processed_images = []
-                    if mask is not None:
-                        p.extra_generation_params["Mask only"] = masking.opts.mask_only if masking.opts.mask_only else None
-                        p.extra_generation_params["Mask auto"] = masking.opts.auto_mask if masking.opts.auto_mask != 'None' else None
-                        p.extra_generation_params["Mask invert"] = masking.opts.invert if masking.opts.invert else None
-                        p.extra_generation_params["Mask blur"] = masking.opts.mask_blur if masking.opts.mask_blur > 0 else None
-                        p.extra_generation_params["Mask erode"] = masking.opts.mask_erode if masking.opts.mask_erode > 0 else None
-                        p.extra_generation_params["Mask dilate"] = masking.opts.mask_dilate if masking.opts.mask_dilate > 0 else None
-                        p.extra_generation_params["Mask model"] = masking.opts.model if masking.opts.model is not None else None
-                        masked_image = masking.run_mask(input_image=input_image, input_mask=mask, return_type='Masked', invert=p.inpainting_mask_invert==1) if mask is not None else input_image
-                    else:
-                        masked_image = input_image
-                    for i, process in enumerate(active_process): # list[image]
-                        debug_log(f'Control: i={i+1} process="{process.processor_id}" input={masked_image} override={process.override}')
-                        processed_image = process(
-                            image_input=masked_image,
-                            mode='RGB',
-                            resize_mode=resize_mode_before,
-                            resize_name=resize_name_before,
-                            scale_tab=selected_scale_tab_before,
-                            scale_by=scale_by_before,
-                        )
-                        if processed_image is not None:
-                            processed_images.append(processed_image)
-                        if shared.opts.control_unload_processor and process.processor_id is not None:
-                            processors.config[process.processor_id]['dirty'] = True # to force reload
-                            process.model = None
-
-                    debug_log(f'Control processed: {len(processed_images)}')
-                    if len(processed_images) > 0:
-                        try:
-                            if len(p.extra_generation_params["Control process"]) == 0:
-                                p.extra_generation_params["Control process"] = None
-                            else:
-                                p.extra_generation_params["Control process"] = ';'.join([p.processor_id for p in active_process if p.processor_id is not None])
-                        except Exception:
-                            pass
-                        if any(img is None for img in processed_images):
-                            if is_generator:
-                                yield terminate('Attempting process but output is none')
-                            return [], '', '', 'Error: output is none'
-                        if len(processed_images) > 1 and len(active_process) != len(active_model):
-                            processed_image = [np.array(i) for i in processed_images]
-                            processed_image = util.blend(processed_image) # blend all processed images into one
-                            processed_image = Image.fromarray(processed_image)
-                            blended_image = processed_image
-                        elif len(processed_images) == 1:
-                            processed_image = processed_images
-                            blended_image = processed_image[0]
-                        else:
-                            blended_image = [np.array(i) for i in processed_images]
-                            blended_image = util.blend(blended_image) # blend all processed images into one
-                            blended_image = Image.fromarray(blended_image)
-                        if isinstance(selected_models, list) and len(processed_images) == len(selected_models):
-                            debug_log(f'Control: inputs match: input={len(processed_images)} models={len(selected_models)}')
-                            p.init_images = processed_images
-                        elif isinstance(selected_models, list) and len(processed_images) != len(selected_models):
-                            if is_generator:
-                                yield terminate(f'Number of inputs does not match: input={len(processed_images)} models={len(selected_models)}')
-                            return [], '', '', 'Error: number of inputs does not match'
-                        elif selected_models is not None:
-                            p.init_images = processed_image
-                    else:
-                        debug_log('Control processed: using input direct')
-                        processed_image = input_image
-
-                    if unit_type == 'reference' and has_models:
-                        p.ref_image = p.override or input_image
-                        p.task_args.pop('image', None)
-                        p.task_args['ref_image'] = p.ref_image
-                        debug_log(f'Control: process=None image={p.ref_image}')
-                        if p.ref_image is None:
-                            if is_generator:
-                                yield terminate('Attempting reference mode but image is none')
-                            return [], '', '', 'Reference mode without image'
-                    elif unit_type == 'controlnet' and has_models:
-                        if input_type == 0: # Control only
-                            if 'control_image' in possible:
-                                p.task_args['control_image'] = [p.init_images] if isinstance(p.init_images, Image.Image) else p.init_images
-                            elif 'image' in possible:
-                                p.task_args['image'] = [p.init_images] if isinstance(p.init_images, Image.Image) else p.init_images
-                            if 'control_mode' in possible:
-                                p.task_args['control_mode'] = getattr(p, 'control_mode', None)
-                            if 'strength' in possible:
-                                p.task_args['strength'] = p.denoising_strength
-                            p.init_images = None
-                        elif input_type == 1: # Init image same as control
-                            p.init_images = [p.override or input_image] * max(1, len(active_model))
-                            if 'inpaint_image' in possible: # flex
-                                p.task_args['inpaint_image'] = p.init_images[0] if isinstance(p.init_images, list) else p.init_images
-                                p.task_args['inpaint_mask'] = Image.new('L', p.task_args['inpaint_image'].size, int(p.denoising_strength * 255))
-                                p.task_args['control_image'] = p.init_images[0] if isinstance(p.init_images, list) else p.init_images
-                                p.task_args['width'] = p.width
-                                p.task_args['height'] = p.height
-                            elif 'control_image' in possible:
-                                p.task_args['control_image'] = p.init_images # switch image and control_image
-                            if 'control_mode' in possible:
-                                p.task_args['control_mode'] = getattr(p, 'control_mode', None)
-                            if 'strength' in possible:
-                                p.task_args['strength'] = p.denoising_strength
-                        elif input_type == 2: # Separate init image
-                            if init_image is None:
-                                shared.log.warning('Control: separate init image not provided')
-                                init_image = input_image
-                            if 'inpaint_image' in possible: # flex
-                                p.task_args['inpaint_image'] = p.init_images[0] if isinstance(p.init_images, list) else p.init_images
-                                p.task_args['inpaint_mask'] = Image.new('L', p.task_args['inpaint_image'].size, int(p.denoising_strength * 255))
-                                p.task_args['control_image'] = p.init_images[0] if isinstance(p.init_images, list) else p.init_images
-                                p.task_args['width'] = p.width
-                                p.task_args['height'] = p.height
-                            elif 'control_image' in possible:
-                                p.task_args['control_image'] = p.init_images # switch image and control_image
-                            if 'control_mode' in possible:
-                                p.task_args['control_mode'] = getattr(p, 'control_mode', None)
-                            if 'strength' in possible:
-                                p.task_args['strength'] = p.denoising_strength
-                            p.init_images = [init_image] * len(active_model)
-                        if hasattr(shared.sd_model, 'controlnet') and hasattr(p.task_args, 'control_image') and len(p.task_args['control_image']) > 1 and (shared.sd_model.__class__.__name__ == 'StableDiffusionXLControlNetUnionPipeline'): # special case for controlnet-union
-                            p.task_args['control_image'] = [[x] for x in p.task_args['control_image']]
-                            p.task_args['control_mode'] = [[x] for x in p.task_args['control_mode']]
-
-                    if is_generator:
-                        image_txt = f'{blended_image.width}x{blended_image.height}' if blended_image is not None else 'None'
-                        msg = f'process | {index} of {frames if video is not None else len(inputs)} | {"Image" if video is None else "Frame"} {image_txt}'
-                        debug_log(f'Control yield: {msg}')
-                        if is_generator:
-                            yield (None, blended_image, f'Control {msg}')
-                    t2 += time.time() - t2
-
-                    # determine txt2img, img2img, inpaint pipeline
-                    if unit_type == 'reference' and has_models: # special case
-                        p.is_control = True
-                        shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
-                    elif not has_models: # run in txt2img/img2img/inpaint mode
-                        if mask is not None:
-                            p.task_args['strength'] = p.denoising_strength
-                            p.image_mask = mask
-                            p.init_images = input_image if isinstance(input_image, list) else [input_image]
-                            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.INPAINTING)
-                        elif processed_image is not None:
-                            p.init_images = processed_image if isinstance(processed_image, list) else [processed_image]
-                            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
-                        else:
-                            p.init_hr(p.scale_by, p.resize_name, force=True)
-                            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
-                    elif has_models: # actual control
-                        p.is_control = True
-                        if mask is not None:
-                            p.task_args['strength'] = denoising_strength
-                            p.image_mask = mask
-                            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.INPAINTING) # only controlnet supports inpaint
-                        if hasattr(p, 'init_images') and p.init_images is not None:
-                            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) # only controlnet supports img2img
-                        else:
-                            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
-                            if hasattr(p, 'init_images') and p.init_images is not None and 'image' in possible:
-                                p.task_args['image'] = p.init_images # need to set explicitly for txt2img
-                                del p.init_images
-                        if unit_type == 'lite':
-                            if input_type == 0:
-                                shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
-                                shared.sd_model.no_task_switch = True
-                            elif input_type == 1:
-                                p.init_images = [input_image]
-                            elif input_type == 2:
-                                if init_image is None:
-                                    shared.log.warning('Control: separate init image not provided')
-                                    init_image = input_image
-                                p.init_images = [init_image]
-                            instance.apply(selected_models, processed_image, control_conditioning)
-                    if hasattr(p, 'init_images') and p.init_images is None: # delete empty
-                        del p.init_images
+                    from modules.control.processor import preprocess_image
+                    processed_image = preprocess_image(p, pipe, input_image, init_image, mask, input_type, unit_type, active_process, active_model, selected_models, has_models)
 
                     # final check
-                    if has_models and shared.sd_model.__class__.__name__ not in unified_models:
-                        if unit_type in ['controlnet', 't2i adapter', 'lite', 'xs'] \
-                            and p.task_args.get('image', None) is None \
-                            and p.task_args.get('control_image', None) is None \
-                            and getattr(p, 'init_images', None) is None \
-                            and getattr(p, 'image', None) is None:
-                            if is_generator:
-                                shared.log.debug(f'Control args: {p.task_args}')
-                                yield terminate(f'Mode={p.extra_generation_params.get("Control type", None)} input image is none')
-                            return [], '', '', 'Error: Input image is none'
-
-                    # resize mask
-                    if mask is not None and resize_mode_mask != 0 and resize_name_mask != 'None':
-                        if selected_scale_tab_mask == 1:
-                            width_mask, height_mask = int(input_image.width * scale_by_mask), int(input_image.height * scale_by_mask)
-                        p.width, p.height = width_mask, height_mask
-                        debug_log(f'Control resize: op=mask image={mask} width={width_mask} height={height_mask} mode={resize_mode_mask} name={resize_name_mask} context="{resize_context_mask}"')
+                    if has_models:
+                        if shared.sd_model.__class__.__name__ not in unified_models:
+                            if unit_type in ['controlnet', 't2i adapter', 'lite', 'xs'] \
+                                and p.task_args.get('image', None) is None \
+                                and p.task_args.get('control_image', None) is None \
+                                and getattr(p, 'init_images', None) is None \
+                                and getattr(p, 'image', None) is None:
+                                if is_generator:
+                                    shared.log.debug(f'Control args: {p.task_args}')
+                                    yield terminate(f'Mode={p.extra_generation_params.get("Control type", None)} input image is none')
+                                return [], '', '', 'Error: Input image is none'
+                        if unit_type == 'lite':
+                            instance.apply(selected_models, processed_image, control_conditioning)
 
                     # pipeline
                     output = None
@@ -736,8 +531,6 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
                             pipe.restore_pipeline = restore_pipeline
                             shared.sd_model.restore_pipeline = restore_pipeline
                         debug_log(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)} class={pipe.__class__}')
-                        # debug_log(f'Control exec pipeline: p={vars(p)}')
-                        # debug_log(f'Control exec pipeline: args={p.task_args} image={p.task_args.get("image", None)} control={p.task_args.get("control_image", None)} mask={p.task_args.get("mask_image", None) or p.image_mask} ref={p.task_args.get("ref_image", None)}')
                         if sd_models.get_diffusers_task(pipe) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: # force vae back to gpu if not in txt2img mode
                             sd_models.move_model(pipe.vae, devices.device)
 
@@ -770,22 +563,12 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
                         # output = pipe(**vars(p)).images # alternative direct pipe exec call
                     else: # blend all processed images and return
                         output = [processed_image]
-                    t3 += time.time() - t3
 
                     # outputs
                     output = output or []
                     for i, output_image in enumerate(output):
                         if output_image is not None:
 
-                            # resize after
-                            is_grid = len(output) == p.batch_size * p.n_iter + 1 and i == 0
-                            if selected_scale_tab_after == 1:
-                                width_after = int(output_image.width * scale_by_after)
-                                height_after = int(output_image.height * scale_by_after)
-                            if resize_mode_after != 0 and resize_name_after != 'None' and not is_grid:
-                                debug_log(f'Control resize: op=after image={output_image} width={width_after} height={height_after} mode={resize_mode_after} name={resize_name_after} context="{resize_context_after}"')
-                                output_image = images.resize_image(resize_mode_after, output_image, width_after, height_after, resize_name_after, context=resize_context_after)
-
                             output_images.append(output_image)
                             if shared.opts.include_mask and not script_run:
                                 if processed_image is not None and isinstance(processed_image, Image.Image):
@@ -810,9 +593,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
             if video is not None:
                 video.release()
 
-            debug_log(f'Control: pipeline units={len(active_model)} process={len(active_process)} time={t3-t0:.2f} init={t1-t0:.2f} proc={t2-t1:.2f} ctrl={t3-t2:.2f} outputs={len(output_images)}')
-            process_timer.add('init', t1-t0)
-            process_timer.add('proc', t2-t1)
+            debug_log(f'Control: pipeline units={len(active_model)} process={len(active_process)} outputs={len(output_images)}')
     except Exception as e:
         shared.log.error(f'Control pipeline failed: type={unit_type} units={len(active_model)} error={e}')
         errors.display(e, 'Control')

modules/control/unit.py

@@ -13,6 +13,7 @@ from modules.control.units import reference # pylint: disable=unused-import
 default_device = None
 default_dtype = None
 unit_types = ['t2i adapter', 'controlnet', 'xs', 'lite', 'reference', 'ip']
+current = []
 
 
 class Unit(): # mashup of gradio controls and mapping to actual implementation classes

modules/control/units/controlnet.py

@@ -382,20 +382,6 @@ class ControlNetPipeline():
                 feature_extractor=getattr(pipeline, 'feature_extractor', None),
                 controlnet=controlnets, # can be a list
             )
-        elif detect.is_sd15(pipeline) and len(controlnets) > 0:
-            from diffusers import StableDiffusionControlNetPipeline
-            self.pipeline = StableDiffusionControlNetPipeline(
-                vae=pipeline.vae,
-                text_encoder=pipeline.text_encoder,
-                tokenizer=pipeline.tokenizer,
-                unet=pipeline.unet,
-                scheduler=pipeline.scheduler,
-                feature_extractor=getattr(pipeline, 'feature_extractor', None),
-                requires_safety_checker=False,
-                safety_checker=None,
-                controlnet=controlnets, # can be a list
-            )
-            sd_models.move_model(self.pipeline, pipeline.device)
         elif detect.is_f1(pipeline) and len(controlnets) > 0:
             from diffusers import FluxControlNetPipeline
             self.pipeline = FluxControlNetPipeline(
@@ -422,6 +408,20 @@ class ControlNetPipeline():
                 scheduler=pipeline.scheduler,
                 controlnet=controlnets, # can be a list
             )
+        elif detect.is_sd15(pipeline) and len(controlnets) > 0:
+            from diffusers import StableDiffusionControlNetPipeline
+            self.pipeline = StableDiffusionControlNetPipeline(
+                vae=pipeline.vae,
+                text_encoder=pipeline.text_encoder,
+                tokenizer=pipeline.tokenizer,
+                unet=pipeline.unet,
+                scheduler=pipeline.scheduler,
+                feature_extractor=getattr(pipeline, 'feature_extractor', None),
+                requires_safety_checker=False,
+                safety_checker=None,
+                controlnet=controlnets, # can be a list
+            )
+            sd_models.move_model(self.pipeline, pipeline.device)
         elif len(loras) > 0:
             self.pipeline = pipeline
             for lora in loras:

modules/control/units/detect.py

@@ -1,51 +1,22 @@
-import diffusers.pipelines as p
-
-
-def is_compatible(model, compatible):
+def is_compatible(model, pattern='None'):
     if model is None:
         return False
     if hasattr(model, '__class__'):
-        return any(model.__class__.__name__ == c.__name__ for c in compatible)
-    return any(isinstance(model, c) for c in compatible)
+        return model.__class__.__name__.startswith(pattern)
+    return False
 
 
 def is_sd15(model):
-    compatible = [
-        p.StableDiffusionPipeline,
-        p.StableDiffusionImg2ImgPipeline,
-        p.StableDiffusionInpaintPipeline,
-        p.StableDiffusionControlNetPipeline,
-    ]
-    return is_compatible(model, compatible)
+    return is_compatible(model, pattern='StableDiffusion')
 
 
 def is_sdxl(model):
-    compatible = [
-        p.StableDiffusionXLPipeline,
-        p.StableDiffusionXLImg2ImgPipeline,
-        p.StableDiffusionXLInpaintPipeline,
-        p.StableDiffusionXLControlNetPipeline,
-        p.StableDiffusionXLControlNetImg2ImgPipeline,
-        p.StableDiffusionXLControlNetUnionPipeline,
-    ]
-    return is_compatible(model, compatible)
+    return is_compatible(model, pattern='StableDiffusionXL')
 
 
 def is_f1(model):
-    compatible = [
-        p.FluxPipeline,
-        p.FluxImg2ImgPipeline,
-        p.FluxInpaintPipeline,
-        p.FluxControlNetPipeline,
-    ]
-    return is_compatible(model, compatible)
+    return is_compatible(model, pattern='Flux')
 
 
 def is_sd3(model):
-    compatible = [
-        p.StableDiffusion3Pipeline,
-        p.StableDiffusion3Img2ImgPipeline,
-        p.StableDiffusion3InpaintPipeline,
-        p.StableDiffusion3ControlNetPipeline,
-    ]
-    return is_compatible(model, compatible)
+    return is_compatible(model, pattern='StableDiffusion3Pipeline')

modules/images_grid.py

@@ -9,6 +9,8 @@ Grid = namedtuple("Grid", ["tiles", "tile_w", "tile_h", "image_w", "image_h", "o
 
 
 def check_grid_size(imgs):
+    if imgs is None or len(imgs) == 0:
+        return False
     mp = 0
     for img in imgs:
         mp += img.width * img.height if img is not None else 0

modules/processing.py

@@ -387,37 +387,52 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                 else:
                     image.info["parameters"] = info
                     output_images.append(image)
-                if shared.opts.samples_save and not p.do_not_save_samples and p.outpath_samples is not None:
-                    info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i)
-                    if isinstance(image, list):
-                        for img in image:
-                            images.save_image(img, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image
-                    else:
-                        images.save_image(image, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image
-                if hasattr(p, 'mask_for_overlay') and p.mask_for_overlay and any([shared.opts.save_mask, shared.opts.save_mask_composite, shared.opts.return_mask, shared.opts.return_mask_composite]):
-                    image_mask = p.mask_for_overlay.convert('RGB')
-                    image1 = image.convert('RGBA').convert('RGBa')
-                    image2 = Image.new('RGBa', image.size)
-                    mask = images.resize_image(3, p.mask_for_overlay, image.width, image.height).convert('L')
-                    image_mask_composite = Image.composite(image1, image2, mask).convert('RGBA')
-                    if shared.opts.save_mask:
-                        images.save_image(image_mask, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p, suffix="-mask")
-                    if shared.opts.save_mask_composite:
-                        images.save_image(image_mask_composite, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p, suffix="-mask-composite")
-                    if shared.opts.return_mask:
-                        output_images.append(image_mask)
-                    if shared.opts.return_mask_composite:
-                        output_images.append(image_mask_composite)
+
+                is_grid = len(output_images) == p.batch_size * p.n_iter + 1 and i == 0
+                for image in output_images:
+                    # resize after
+                    if p.selected_scale_tab_after == 1:
+                        p.width_after, p.height_after = int(image.width * p.scale_by_after), int(image.height * p.scale_by_after)
+                    if p.resize_mode_after != 0 and p.resize_name_after != 'None' and not is_grid:
+                        image = images.resize_image(p.resize_mode_after, image, p.width_after, p.height_after, p.resize_name_after, context=p.resize_context_after)
+
+                    # save images
+                    if shared.opts.samples_save and not p.do_not_save_samples and p.outpath_samples is not None:
+                        info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i)
+                        if isinstance(image, list):
+                            for img in image:
+                                images.save_image(img, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image
+                        else:
+                            images.save_image(image, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image
+
+                    # add masks
+                    if shared.opts.include_mask and not script_run:
+                        if processed_image is not None and isinstance(processed_image, Image.Image):
+                            output_images.append(processed_image)
+                    if hasattr(p, 'mask_for_overlay') and p.mask_for_overlay and any([shared.opts.save_mask, shared.opts.save_mask_composite, shared.opts.return_mask, shared.opts.return_mask_composite]):
+                        image_mask = p.mask_for_overlay.convert('RGB')
+                        image1 = image.convert('RGBA').convert('RGBa')
+                        image2 = Image.new('RGBa', image.size)
+                        mask = images.resize_image(3, p.mask_for_overlay, image.width, image.height).convert('L')
+                        image_mask_composite = Image.composite(image1, image2, mask).convert('RGBA')
+                        if shared.opts.save_mask:
+                            images.save_image(image_mask, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p, suffix="-mask")
+                        if shared.opts.save_mask_composite:
+                            images.save_image(image_mask_composite, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p, suffix="-mask-composite")
+                        if shared.opts.return_mask:
+                            output_images.append(image_mask)
+                        if shared.opts.return_mask_composite:
+                            output_images.append(image_mask_composite)
 
             timer.process.record('post')
             del samples
 
             devices.torch_gc()
 
-        # if not p.xyz:
-        if hasattr(shared.sd_model, 'restore_pipeline') and (shared.sd_model.restore_pipeline is not None):
-            shared.sd_model.restore_pipeline()
-        shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
+        if not p.xyz:
+            if hasattr(shared.sd_model, 'restore_pipeline') and (shared.sd_model.restore_pipeline is not None):
+                shared.sd_model.restore_pipeline()
+            shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
 
         t1 = time.time()
 

modules/processing_class.py

@@ -71,14 +71,36 @@ class StableDiffusionProcessing:
                  hdr_tint_ratio: float = 0,
                  # img2img
                  init_images: list = None,
-                 resize_mode: int = 0,
-                 resize_name: str = 'None',
-                 resize_context: str = 'None',
                  denoising_strength: float = 0.3,
                  image_cfg_scale: float = None,
                  initial_noise_multiplier: float = None, # pylint: disable=unused-argument # a1111 compatibility
+                 # resize
                  scale_by: float = 1,
                  selected_scale_tab: int = 0, # pylint: disable=unused-argument # a1111 compatibility
+                 resize_mode: int = 0,
+                 resize_name: str = 'None',
+                 resize_context: str = 'None',
+                 width_before:int = 0,
+                 width_after:int = 0,
+                 width_mask:int = 0,
+                 height_before:int = 0,
+                 height_after:int = 0,
+                 height_mask:int = 0,
+                 resize_name_before: str = 'None',
+                 resize_name_after: str = 'None',
+                 resize_name_mask: str = 'None',
+                 resize_mode_before: int = 0,
+                 resize_mode_after: int = 0,
+                 resize_mode_mask: int = 0,
+                 resize_context_before: str = 'None',
+                 resize_context_after: str = 'None',
+                 resize_context_mask: str = 'None',
+                 selected_scale_tab_before: int = 0,
+                 selected_scale_tab_after: int = 0,
+                 selected_scale_tab_mask: int = 0,
+                 scale_by_before: float = 1,
+                 scale_by_after: float = 1,
+                 scale_by_mask: float = 1,
                  # inpaint
                  mask: Any = None,
                  latent_mask: Any = None,
@@ -231,6 +253,27 @@ class StableDiffusionProcessing:
         self.mask_for_overlay = mask_for_overlay
         self.paste_to = paste_to
         self.init_latent = None
+        self.width_before = width_before
+        self.width_after = width_after
+        self.width_mask = width_mask
+        self.height_before = height_before
+        self.height_after = height_after
+        self.height_mask = height_mask
+        self.resize_name_before = resize_name_before
+        self.resize_name_after = resize_name_after
+        self.resize_name_mask = resize_name_mask
+        self.resize_mode_before = resize_mode_before
+        self.resize_mode_after = resize_mode_after
+        self.resize_mode_mask = resize_mode_mask
+        self.resize_context_before = resize_context_before
+        self.resize_context_after = resize_context_after
+        self.resize_context_mask = resize_context_mask
+        self.selected_scale_tab_before = selected_scale_tab_before
+        self.selected_scale_tab_after = selected_scale_tab_after
+        self.selected_scale_tab_mask = selected_scale_tab_mask
+        self.scale_by_before = scale_by_before
+        self.scale_by_after = scale_by_after
+        self.scale_by_mask = scale_by_mask
 
         # special handled items
         if firstphase_width != 0 or firstphase_height != 0:
@@ -405,7 +448,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
         super().__init__(**kwargs)
 
     def init(self, all_prompts=None, all_seeds=None, all_subseeds=None):
-        if hasattr(self, 'init_images') and self.init_images is not None and len(self.init_images) > 0:
+        if self.init_images is not None and len(self.init_images) > 0:
             if self.width is None or self.width == 0:
                 self.width = int(8 * (self.init_images[0].width * self.scale_by // 8))
             if self.height is None or self.height == 0:
@@ -423,7 +466,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
             self.all_subseeds = all_subseeds
         if self.image_mask is not None:
             self.ops.append('inpaint')
-        elif hasattr(self, 'init_images') and self.init_images is not None:
+        elif self.init_images is not None and len(self.init_images) > 0:
             self.ops.append('img2img')
         crop_region = None
 
@@ -456,17 +499,16 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
         if add_color_corrections:
             self.color_corrections = []
         processed_images = []
-        if getattr(self, 'init_images', None) is None:
+        if self.init_images is None:
             return
         if not isinstance(self.init_images, list):
             self.init_images = [self.init_images]
         for img in self.init_images:
             if img is None:
-                # shared.log.warning(f"Skipping empty image: images={self.init_images}")
                 continue
-            self.init_img_hash = hashlib.sha256(img.tobytes()).hexdigest()[0:8] # pylint: disable=attribute-defined-outside-init
-            self.init_img_width = img.width # pylint: disable=attribute-defined-outside-init
-            self.init_img_height = img.height # pylint: disable=attribute-defined-outside-init
+            self.init_img_hash = getattr(self, 'init_img_hash', hashlib.sha256(img.tobytes()).hexdigest()[0:8]) # pylint: disable=attribute-defined-outside-init
+            self.init_img_width = getattr(self, 'init_img_width', img.width) # pylint: disable=attribute-defined-outside-init
+            self.init_img_height = getattr(self, 'init_img_height', img.height) # pylint: disable=attribute-defined-outside-init
             if shared.opts.save_init_img:
                 images.save_image(img, path=shared.opts.outdir_init_images, basename=None, forced_filename=self.init_img_hash, suffix="-init-image")
             image = images.flatten(img, shared.opts.img2img_background_color)

modules/processing_diffusers.py

@@ -441,10 +441,10 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
         return results
 
     # sanitize init_images
-    if hasattr(p, 'init_images') and getattr(p, 'init_images', None) is None:
-        del p.init_images
     if hasattr(p, 'init_images') and not isinstance(getattr(p, 'init_images', []), list):
         p.init_images = [p.init_images]
+    if hasattr(p, 'init_images') and isinstance(getattr(p, 'init_images', []), list):
+        p.init_images = [i for i in p.init_images if i is not None]
     if len(getattr(p, 'init_images', [])) > 0:
         while len(p.init_images) < len(p.prompts):
             p.init_images.append(p.init_images[-1])

modules/processing_info.py

@@ -90,19 +90,21 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
         is_resize = p.hr_resize_mode > 0 and (p.hr_upscaler != 'None' or p.hr_resize_mode == 5)
         is_fixed = p.hr_resize_x > 0 or p.hr_resize_y > 0
         args["Refine"] = p.enable_hr
-        args["Hires force"] = p.hr_force
-        args["Hires steps"] = p.hr_second_pass_steps
-        args["HiRes mode"] = p.hr_resize_mode if is_resize else None
-        args["HiRes context"] = p.hr_resize_context if p.hr_resize_mode == 5 else None
-        args["Hires upscaler"] = p.hr_upscaler if is_resize else None
-        if is_fixed:
-            args["Hires fixed"] = f"{p.hr_resize_x}x{p.hr_resize_y}" if is_resize else None
-        else:
-            args["Hires scale"] = p.hr_scale if is_resize else None
-        args["Hires size"] = f"{p.hr_upscale_to_x}x{p.hr_upscale_to_y}" if is_resize else None
-        args["Hires strength"] = p.denoising_strength
-        args["Hires sampler"] = p.hr_sampler_name if p.hr_sampler_name != p.sampler_name else None
-        args["Hires CFG scale"] = p.image_cfg_scale
+        if is_resize:
+            args["HiRes mode"] = p.hr_resize_mode
+            args["HiRes context"] = p.hr_resize_context if p.hr_resize_mode == 5 else None
+            args["Hires upscaler"] = p.hr_upscaler
+            if is_fixed:
+                args["Hires fixed"] = f"{p.hr_resize_x}x{p.hr_resize_y}"
+            else:
+                args["Hires scale"] = p.hr_scale
+            args["Hires size"] = f"{p.hr_upscale_to_x}x{p.hr_upscale_to_y}"
+        if p.hr_force or ('Latent' in p.hr_upscaler):
+            args["Hires force"] = p.hr_force
+            args["Hires steps"] = p.hr_second_pass_steps
+            args["Hires strength"] = p.denoising_strength
+            args["Hires sampler"] = p.hr_sampler_name if p.hr_sampler_name != p.sampler_name else None
+            args["Hires CFG scale"] = p.image_cfg_scale
     if 'refine' in p.ops:
         args["Refine"] = p.enable_hr
         args["Refiner"] = None if (not shared.opts.add_model_name_to_info) or (not shared.sd_refiner) or (not shared.sd_refiner.sd_checkpoint_info.model_name) else shared.sd_refiner.sd_checkpoint_info.model_name.replace(',', '').replace(':', '')
@@ -123,23 +125,21 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
         # lookup by index
         if getattr(p, 'resize_mode', None) is not None:
             args['Resize mode'] = shared.resize_modes[p.resize_mode] if shared.resize_modes[p.resize_mode] != 'None' else None
-    if hasattr(p, 'width_before') and hasattr(p, 'height_before'):
-        args['Size'] = f"{p.width_before}x{p.height_before}" # override size
-        if getattr(p, 'resize_mode_before', None) is not None:
-            args['Size before'] = f"{p.width_before}x{p.height_before}"
-            args['Size mode before'] = p.resize_mode_before
-            args['Size scale before'] = p.scale_by_before if p.scale_by_before != 1.0 else None
-            args['Size name before'] = p.resize_name_before
-    if getattr(p, 'resize_mode_after', None) is not None:
-        args['Size after'] = f"{p.width_after}x{p.height_after}" if hasattr(p, 'width_after') and hasattr(p, 'height_after') else None
-        args['Size mode after'] = p.resize_mode_after
-        args['Size scale after'] = p.scale_by_after if p.scale_by_after != 1.0 else None
-        args['Size name after'] = p.resize_name_after
-    if getattr(p, 'resize_mode_mask', None) is not None:
-        args['Size mask'] = f"{p.width_mask}x{p.height_mask}" if hasattr(p, 'width_mask') and hasattr(p, 'height_mask') else None
-        args['Size mode mask'] = p.resize_mode_mask
-        args['Size scale mask'] = p.scale_by_mask
-        args['Size name mask'] = p.resize_name_mask
+    if p.resize_mode_before != 0 and p.resize_name_before != 'None' and hasattr(p, 'init_images') and p.init_images is not None and len(p.init_images) > 0:
+        args['Resize before'] = f"{p.width_before}x{p.height_before}"
+        args['Resize mode before'] = p.resize_mode_before
+        args['Resize name before'] = p.resize_name_before
+        args['Resize scale before'] = p.scale_by_before if p.scale_by_before != 1.0 else None
+    if p.resize_mode_after != 0 and p.resize_name_after != 'None':
+        args['Resize after'] = f"{p.width_after}x{p.height_after}"
+        args['Resize mode after'] = p.resize_mode_after
+        args['Resize name after'] = p.resize_name_after
+        args['Resize scale after'] = p.scale_by_after if p.scale_by_after != 1.0 else None
+    if p.resize_name_mask != 'None' and p.scale_by_mask != 1.0:
+        args['Resize mask'] = f"{p.width_mask}x{p.height_mask}"
+        args['Resize mode mask'] = p.resize_mode_mask
+        args['Resize name mask'] = p.resize_name_mask
+        args['Resize scale mask'] = p.scale_by_mask
     if 'detailer' in p.ops:
         args["Detailer"] = ', '.join(shared.opts.detailer_models)
         args["Detailer steps"] = p.detailer_steps

scripts/xyz/xyz_grid_classes.py

@@ -41,7 +41,7 @@ from scripts.xyz.xyz_grid_shared import (
  ) # pylint: disable=no-name-in-module, unused-import
 from modules import shared, shared_items, sd_samplers, ipadapter, sd_models, sd_vae, sd_unet
 from modules.control.units import controlnet, t2iadapter
-from modules.control.processors_list import processors
+from modules.control import processors, processor
 
 
 class AxisOption:
@@ -259,7 +259,7 @@ axis_options = [
     AxisOption("[IP adapter] Ends", float, apply_field('ip_adapter_ends')),
     AxisOption("[Control] ControlNet", str, apply_control('controlnet'), cost=0.9, choices=lambda: list(controlnet.all_models)),
     AxisOption("[Control] T2IAdapter", str, apply_control('t2i adapter'), cost=0.9, choices=lambda: list(t2iadapter.all_models)),
-    AxisOption("[Control] Processor", str, apply_control('processor'), cost=2.0, choices=lambda: processors),
+    AxisOption("[Control] Processor", str, apply_control('processor'), cost=2.0, choices=lambda: processor.processors),
     AxisOption("[Control] Strength", float, apply_control('control_strength')),
     AxisOption("[Control] Start", float, apply_control('control_start')),
     AxisOption("[Control] End", float, apply_control('control_end')),