From e2a443c8859daa8173909b6f4a0e368d771398c3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 16 Jul 2025 12:07:43 -0400 Subject: [PATCH] control xyz support Signed-off-by: Vladimir Mandic --- modules/control/processor.py | 18 ++++----- modules/control/run.py | 3 +- modules/control/unit.py | 4 +- scripts/xyz/xyz_grid_classes.py | 2 +- scripts/xyz/xyz_grid_shared.py | 68 +++++++++++++++++++-------------- 5 files changed, 53 insertions(+), 42 deletions(-) diff --git a/modules/control/processor.py b/modules/control/processor.py index 844fce49b..d6f0e1a62 100644 --- a/modules/control/processor.py +++ b/modules/control/processor.py @@ -39,15 +39,15 @@ processors = [ def preprocess_image( p:StableDiffusionProcessingControl, pipe, - input_image:Image.Image, - init_image:Image.Image, - input_mask:Image.Image, - input_type:str, - unit_type:str, - active_process:list, - active_model:list, - selected_models:list, - has_models:bool, + input_image:Image.Image = None, + init_image:Image.Image = None, + input_mask:Image.Image = None, + input_type:str = 0, + unit_type:str = 'controlnet', + active_process:list = [], + active_model:list = [], + selected_models:list = [], + has_models:bool = False, ): t0 = time.time() diff --git a/modules/control/run.py b/modules/control/run.py index 0b7b9b741..3fae897a3 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -12,9 +12,9 @@ from modules.control.units import xs # VisLearn ControlNet-XS from modules.control.units import lite # Kohya ControlLLLite from modules.control.units import t2iadapter # TencentARC T2I-Adapter from modules.control.units import reference # ControlNet-Reference +from modules.control.processor import preprocess_image from modules import devices, shared, errors, processing, images, sd_models, scripts_manager, masking from modules.processing_class import StableDiffusionProcessingControl -from modules.processing_info import create_infotext from modules.ui_common import infotext_to_html from modules.api import script @@ -505,7 +505,6 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg continue index += 1 - from modules.control.processor import preprocess_image processed_image = preprocess_image(p, pipe, input_image, init_image, mask, input_type, unit_type, active_process, active_model, selected_models, has_models) # final check diff --git a/modules/control/unit.py b/modules/control/unit.py index 94161c56d..da13f35aa 100644 --- a/modules/control/unit.py +++ b/modules/control/unit.py @@ -59,6 +59,8 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c result_txt = None, extra_controls: list = [], ): + self.model_id = model_id + self.process_id = process_id self.controls = [gr.Label(value=unit_type, visible=False)] # separator self.index = index self.enabled = enabled or False @@ -81,8 +83,6 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c # global settings but passed per-unit self.factor = 1.0 self.guess = False - self.start = 0 - self.end = 1 # reference settings self.attention = 'Attention' self.fidelity = 0.5 diff --git a/scripts/xyz/xyz_grid_classes.py b/scripts/xyz/xyz_grid_classes.py index b42a429ad..338a8752e 100644 --- a/scripts/xyz/xyz_grid_classes.py +++ b/scripts/xyz/xyz_grid_classes.py @@ -259,7 +259,7 @@ axis_options = [ AxisOption("[IP adapter] Ends", float, apply_field('ip_adapter_ends')), AxisOption("[Control] ControlNet", str, apply_control('controlnet'), cost=0.9, choices=lambda: list(controlnet.all_models)), AxisOption("[Control] T2IAdapter", str, apply_control('t2i adapter'), cost=0.9, choices=lambda: list(t2iadapter.all_models)), - AxisOption("[Control] Processor", str, apply_control('processor'), cost=2.0, choices=lambda: processor.processors), + AxisOption("[Control] Processor", str, apply_control('processor'), cost=0.6, choices=lambda: processor.processors), AxisOption("[Control] Strength", float, apply_control('control_strength')), AxisOption("[Control] Start", float, apply_control('control_start')), AxisOption("[Control] End", float, apply_control('control_end')), diff --git a/scripts/xyz/xyz_grid_shared.py b/scripts/xyz/xyz_grid_shared.py index 9203c3168..2f53b4422 100644 --- a/scripts/xyz/xyz_grid_shared.py +++ b/scripts/xyz/xyz_grid_shared.py @@ -289,44 +289,56 @@ def apply_detailer(p, opt, x): def apply_control(field): def fun(p, x, xs): - shared.log.debug(f'XYZ grid apply control: {field}={x}') - if field in ['controlnet', 't2i adapter']: - from modules.control import run - vals = x.split(':') - model_id = vals[0].strip() if len(vals) > 0 else None - process_id = vals[1].strip() if len(vals) > 1 else None - strength = float(vals[2].strip()) if len(vals) > 2 else 1.0 - start = float(vals[3].strip()) if len(vals) > 3 else 0.0 - end = float(vals[4].strip()) if len(vals) > 4 else 1.0 + if getattr(p, 'xyz_init_images', None) is not None and len(getattr(p, 'xyz_init_images', [])) > 0: # backup init images since they get modified + p.init_images = getattr(p, 'xyz_init_images', None) + else: + p.xyz_init_images = getattr(p, 'init_images', None) + if getattr(p, 'init_images', None) is None or len(getattr(p, 'init_images', [])) == 0: + shared.log.error(f'XYZ grid apply control: init image is required') + return x + if field in ['controlnet', 't2i adapter', 'processor']: + from modules.control import run, processor + unit_type = 'controlnet' # set default + if field in ['controlnet', 't2i adapter']: + unit_type = field + model_id = x + process_id = run.unit.current[0].process_id if len(run.unit.current) > 0 else None + elif field == 'processor': + model_id = run.unit.current[0].model_id if len(run.unit.current) > 0 else None + process_id = x + start = run.unit.current[0].start if len(run.unit.current) > 0 else 0 + end = run.unit.current[0].end if len(run.unit.current) > 0 else 1.0 + strength = run.unit.current[0].model_strength if len(run.unit.current) > 0 else 1.0 unit = run.unit.Unit( - index=0, - enabled=True, - unit_type=field, - model_id=model_id, - process_id=process_id, - strength=strength, - start=start, - end=end, + index = 0, + enabled = True, + unit_type = unit_type, + model_id = getattr(model_id, 'value', model_id), # gradio-component-to-string + process_id = getattr(process_id, 'value', process_id), + start = getattr(start, 'value', start), + end = getattr(end, 'value', end), + strength = getattr(strength, 'value', strength), ) - run.init_units([unit]) - active_process, active_model, active_strength, active_start, active_end = run.check_active(p, unit.type, [unit]) - has_models, selected_models, control_conditioning, control_guidance_start, control_guidance_end = run.check_enabled(p, unit.type, [unit], active_model, active_strength, active_start, active_end) + shared.log.debug(f'XYZ grid apply control: {field}="{x}" unit={unit}') + if len(run.unit.current) > 0: + run.unit.current[0] = unit + else: + run.unit.current = [unit] + run.init_units(run.unit.current) + active_process, active_model, active_strength, active_start, active_end = run.check_active(p, unit.type, run.unit.current) + has_models, selected_models, control_conditioning, control_guidance_start, control_guidance_end = run.check_enabled(p, unit.type, run.unit.current, active_model, active_strength, active_start, active_end) pipe = run.set_pipe(p, has_models, unit.type, selected_models, active_model, active_strength, control_conditioning, control_guidance_start, control_guidance_end) + processed_image = processor.preprocess_image(p, pipe, input_image=p.init_images[0], unit_type=unit.type, active_process=active_process, active_model=active_model, selected_models=selected_models, has_models=has_models) if pipe is not None: shared.sd_model = pipe - elif field == 'processor': - from modules.control.processors import Processor - processor = Processor(x) - if processor is not None: - processor.reset() - # p.task_args['image'] = [processor(p.init_images)] - p.task_args['image'] = processor(p.init_images) - p.init_images = None elif field == 'control_start': + shared.log.debug(f'XYZ grid apply control: {field}={x}') p.task_args['control_guidance_start'] = float(x) elif field == 'control_end': + shared.log.debug(f'XYZ grid apply control: {field}={x}') p.task_args['control_guidance_end'] = float(x) elif field == 'control_strength': + shared.log.debug(f'XYZ grid apply control: {field}={x}') p.task_args['adapter_conditioning_scale'] = float(x) p.task_args['controlnet_conditioning_scale'] = float(x) return fun