From eaf06fbc74de85e4076555ce9574d1eda12900bb Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 30 Apr 2026 09:30:11 +0200 Subject: [PATCH] all direct input images Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 20 ++-- extensions-builtin/sdnext-kanvas | 2 +- extensions-builtin/sdnext-modernui | 2 +- modules/control/run.py | 151 ++++++++++++++++++----------- modules/processing_args.py | 5 +- modules/sd_models.py | 2 +- pipelines/model_google.py | 1 + 7 files changed, 113 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c17c8c085..03fe52caa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,25 @@ # Change Log for SD.Next -## Update for 2026-04-29 +## TODO +- Tag multi-image pipes with `use_images_direct` + +## Update for 2026-04-30 + +- **Features** + - **Multi-image** workflows! + for models that support multiple images as inputs, you can now add multiple stages in Kanvas + prompts like "*place character from first image, add background from second image, render in style from third image*" are now possible +- **UI** + - add button to manually reorient input/output panels + - all ui panels can be minimized/maximized by clicking on their header + state is preserved across sessions and can be used to hide rarely used panels and declutter the workspace + - **Kanvas** re-order stages by clicking on active stage - **Control** - remove buttons: *input/control/process* - move params *control input type* to control menu section - remove "processed preview" from ui preprocessor output can still be generated by clicking preview button in in control unit and it will render into normal output area -- **UI** - - all ui panels can be minimized/maximized by clicking on their header - state is preserved across sessions and can be used to hide rarely used panels and declutter the workspace -- **Kanvas** - - re-order stages by clicking on active stage ## Update for 2026-04-28 diff --git a/extensions-builtin/sdnext-kanvas b/extensions-builtin/sdnext-kanvas index 5c71a0144..68a9564f7 160000 --- a/extensions-builtin/sdnext-kanvas +++ b/extensions-builtin/sdnext-kanvas @@ -1 +1 @@ -Subproject commit 5c71a01449fe00fcc36525b2a10fe4b90b86731e +Subproject commit 68a9564f7c9bfd96993341217ff18a1266ff8960 diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index e282300b3..2c1dc64a0 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit e282300b3d109580f0b32da1dcdf18c7c8504186 +Subproject commit 2c1dc64a085cae33cfd67e7406fb07f564dd27d2 diff --git a/modules/control/run.py b/modules/control/run.py index c201367f5..78e23272b 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -274,6 +274,68 @@ def init_units(units: list[unit.Unit]): u.process.override = u.override +def control_process(p: StableDiffusionProcessingControl, + input_script_args: list | None = None, + override_script_name: str | None = None, + override_script_args: list | None = None, + input_image: Image.Image = None, # only used for tiling, otherwise processor.preprocess_image set p params + ): + debug_log(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)} class={pipe.__class__}') + if sd_models.get_diffusers_task(pipe) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: # force vae back to gpu if not in txt2img mode + sd_models.move_model(pipe.vae, devices.device) + + # what are we doing? + if 'control' in p.ops: + p.outpath_samples = resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_control_samples) + elif 'img2img' in p.ops: + p.outpath_samples = resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_img2img_samples) + elif 'txt2img' in p.ops: + p.outpath_samples = resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_txt2img_samples) + else: # fallback to txt2img + p.outpath_samples = resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_txt2img_samples) + + # init scripts + p.scripts = scripts_manager.scripts_control + p.script_args = input_script_args or [] + if len(p.script_args) == 0: + if not p.scripts: + p.scripts.initialize_scripts(False) + p.script_args = script.init_default_script_args(p.scripts) + + # init override scripts + if override_script_name and override_script_args and len(override_script_name) > 0: + selectable_scripts, selectable_script_idx = script.get_selectable_script(override_script_name, p.scripts) + if selectable_scripts: + for idx in range(len(override_script_args)): + p.script_args[selectable_scripts.args_from + idx] = override_script_args[idx] + p.script_args[0] = selectable_script_idx + 1 + + # actual processing + script_run = False + processed: processing.Processed = None + if p.is_tile: + processed: processing.Processed = tile.run_tiling(p, input_image) + if processed is None and p.scripts is not None: + processed = p.scripts.run(p, *p.script_args) + if processed is None: + processed: processing.Processed = processing.process_images(p) # run actual pipeline + else: + script_run = True + + # postprocessing + if p.scripts is not None: + processed = p.scripts.after(p, processed, *p.script_args) + + output = None + info = None + if processed is not None and processed.images is not None: + output = processed.images + info = [processed.infotext(p, i) for i in range(len(output))] + + # output = pipe(**vars(p)).images # debug: direct pipe exec call + return output, info, script_run + + def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg units: list[unit.Unit] | None = None, inputs: list[Image.Image] | None = None, inits: list[Image.Image] | None = None, mask: Image.Image = None, unit_type: str | None = None, is_generator: bool = True, input_type: int = 0, @@ -633,17 +695,20 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg processed_image = None if frame is not None: inputs = [Image.fromarray(frame)] # cv2 to pil - for i, input_image in enumerate(inputs): - if input_image is not None: - p.ops.append('img2img') + for i, input_image in enumerate(inputs): # loop per-input, but with early-break if pipe is None: # pipe may have been reset externally if video is None: break # non-video: pipeline was consumed, no need to re-process remaining inputs pipe = set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, active_units, control_conditioning, control_guidance_start, control_guidance_end, inits) debug_log(f'Control pipeline reinit: class={pipe.__class__.__name__}') + pipe.restore_pipeline = restore_pipeline shared.sd_model.restore_pipeline = restore_pipeline debug_log(f'Control Control image: {i + 1} of {len(inputs)}') + + if input_image is not None: + p.ops.append('img2img') + if shared.state.skipped: shared.state.skipped = False continue @@ -652,6 +717,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg if is_generator: yield terminate('Interrupted') return terminate('Interrupted') + # get input if isinstance(input_image, str) and os.path.exists(input_image): try: @@ -664,7 +730,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg debug_log('Control Init image: same as control') init_image = input_image elif inits is None or len(inits) == 0: - debug_log('Control Init image: none') + debug_log('Control init image: none') init_image = None elif len(inits) > i and isinstance(inits[i], str): debug_log(f'Control: init image: {inits[i]}') @@ -683,7 +749,22 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg continue index += 1 - processed_image, blended_image = preprocess_image(p, pipe, input_image, init_image, mask, input_type, unit_type, active_process, active_model, selected_models, has_models, active_units) + if getattr(pipe, 'use_images_direct', False): + p.init_images = inputs + else: + processed_image, blended_image = preprocess_image(p, + pipe, + input_image, + init_image, + mask, + input_type, + unit_type, + active_process, + active_model, + selected_models, + has_models, + active_units, + ) if is_generator: yield (None, blended_image, '') # result is control_output, proces_output @@ -701,62 +782,18 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg if unit_type == 'lite': instance.apply(selected_models, processed_image, control_conditioning) - # what are we doing? - if 'control' in p.ops: - p.outpath_samples = resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_control_samples) - elif 'img2img' in p.ops: - p.outpath_samples = resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_img2img_samples) - elif 'txt2img' in p.ops: - p.outpath_samples = resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_txt2img_samples) - else: # fallback to txt2img - p.outpath_samples = resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_txt2img_samples) - # pipeline output = None script_run = False - if pipe is not None: # run new pipeline - debug_log(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)} class={pipe.__class__}') - if sd_models.get_diffusers_task(pipe) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: # force vae back to gpu if not in txt2img mode - sd_models.move_model(pipe.vae, devices.device) - - # init scripts - p.scripts = scripts_manager.scripts_control - p.script_args = input_script_args or [] - if len(p.script_args) == 0: - if not p.scripts: - p.scripts.initialize_scripts(False) - p.script_args = script.init_default_script_args(p.scripts) - - # init override scripts - if override_script_name and override_script_args and len(override_script_name) > 0: - selectable_scripts, selectable_script_idx = script.get_selectable_script(override_script_name, p.scripts) - if selectable_scripts: - for idx in range(len(override_script_args)): - p.script_args[selectable_scripts.args_from + idx] = override_script_args[idx] - p.script_args[0] = selectable_script_idx + 1 - - # actual processing - processed: processing.Processed = None - if p.is_tile: - processed: processing.Processed = tile.run_tiling(p, input_image) - if processed is None and p.scripts is not None: - processed = p.scripts.run(p, *p.script_args) - if processed is None: - processed: processing.Processed = processing.process_images(p) # run actual pipeline - else: - script_run = True - - # postprocessing - if p.scripts is not None: - processed = p.scripts.after(p, processed, *p.script_args) - output = None - if processed is not None and processed.images is not None: - output = processed.images - info_txt = [processed.infotext(p, i) for i in range(len(output))] - - # output = pipe(**vars(p)).images # alternative direct pipe exec call - else: # blend all processed images and return + if pipe is None: # blend all processed images and return output = processed_image + else: # run new pipeline + output, info_txt, script_run = control_process(p, + input_script_args, + override_script_name, + override_script_args, + input_image, + ) # outputs output = output or [] diff --git a/modules/processing_args.py b/modules/processing_args.py index fc7ba7e43..b0d0e0361 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -156,10 +156,7 @@ def task_specific_kwargs(p, model): if ('WanVACEPipeline' in model_cls) and (p.init_images is not None) and (len(p.init_images) > 0): task_args['reference_images'] = p.init_images if ('GoogleNanoBananaPipeline' in model_cls) and (p.init_images is not None) and (len(p.init_images) > 0): - if hasattr(p, 'orig_init_images') and (p.orig_init_images is not None) and len(p.orig_init_images) > 0: - task_args['images'] = p.orig_init_images - else: - task_args['images'] = p.init_images + task_args['images'] = p.init_images if ('GlmImagePipeline' in model_cls) and (p.init_images is not None) and (len(p.init_images) > 0): task_args['image'] = p.init_images if 'BlipDiffusionPipeline' in model_cls: diff --git a/modules/sd_models.py b/modules/sd_models.py index 296f32d5b..5e9e847d2 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1290,7 +1290,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): add_noise_pred_to_diffusers_callback(new_pipe.pipe) fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access - log.debug(f"Pipeline class change: original={cls} target={new_pipe.__class__.__name__} device={pipe.device} fn={fn}") # pylint: disable=protected-access + log.debug(f"Pipeline class change: source={cls} target={new_pipe.__class__.__name__} device={pipe.device} fn={fn}") # pylint: disable=protected-access if shared.opts.diffusers_offload_mode == 'none': move_model(new_pipe, pipe.device) diff --git a/pipelines/model_google.py b/pipelines/model_google.py index 62007cc6b..9bb062811 100644 --- a/pipelines/model_google.py +++ b/pipelines/model_google.py @@ -40,6 +40,7 @@ def get_size_buckets(width: int, height: int) -> tuple[str, str]: class GoogleNanoBananaPipeline(): def __init__(self, model_name: str): + self.use_images_direct = True self.model = model_name self.client = None self.config = None