From 93a3b532d8d0216d1eab4dc992fa78401a71ca89 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 22 Dec 2023 14:15:08 -0500 Subject: [PATCH] control input control --- CHANGELOG.md | 2 +- javascript/sdnext.css | 2 + modules/control/adapters.py | 4 +- modules/control/controlnets.py | 4 +- modules/control/controlnetsxs.py | 4 +- modules/control/processors.py | 10 ++- modules/control/run.py | 123 ++++++++++++++++++-------- modules/control/unit.py | 9 +- modules/ui.py | 7 +- modules/ui_control.py | 144 ++++++++++++++++++++++--------- wiki | 2 +- 11 files changed, 216 insertions(+), 95 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71af9e0c3..8a2d44611 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2023-12-21 +## Update for 2023-12-22 *Note*: based on `diffusers==0.25.0.dev0` diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 846c4c5f7..b4d7fce93 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -256,6 +256,8 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt font-size: 0.7em; z-index: 50; font-family: monospace; display: none; } /* control */ +#control_input_type { max-width: 18em } +#control_settings .small-accordion .form { min-width: 350px !important } .control-button { min-height: 42px; max-height: 42px; line-height: 1em; } .control-tabs>.tab-nav { margin-bottom: 0; margin-top: 0; } .processor-settings { padding: 0 !important; max-width: 300px; } diff --git a/modules/control/adapters.py b/modules/control/adapters.py index 9e8360567..12ed28907 100644 --- a/modules/control/adapters.py +++ b/modules/control/adapters.py @@ -77,7 +77,7 @@ class Adapter(): self.model = None self.model_id = None - def load(self, model_id: str = None): + def load(self, model_id: str = None) -> str: try: t0 = time.time() model_id = model_id or self.model_id @@ -97,9 +97,11 @@ class Adapter(): t1 = time.time() self.model_id = model_id log.debug(f'Control {what} loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}') + return f'{what} loaded model: {model_id}' except Exception as e: log.error(f'Control {what} model load failed: id="{model_id}" error={e}') errors.display(e, f'Control {what} load') + return f'{what} failed to load model: {model_id}' class AdapterPipeline(): diff --git a/modules/control/controlnets.py b/modules/control/controlnets.py index 14a73c2cf..0230fe773 100644 --- a/modules/control/controlnets.py +++ b/modules/control/controlnets.py @@ -90,7 +90,7 @@ class ControlNet(): self.model = None self.model_id = None - def load(self, model_id: str = None): + def load(self, model_id: str = None) -> str: try: t0 = time.time() model_id = model_id or self.model_id @@ -115,9 +115,11 @@ class ControlNet(): t1 = time.time() self.model_id = model_id log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}') + return f'{what} loaded model: {model_id}' except Exception as e: log.error(f'Control {what} model load failed: id="{model_id}" error={e}') errors.display(e, f'Control {what} load') + return f'{what} failed to load model: {model_id}' class ControlNetPipeline(): diff --git a/modules/control/controlnetsxs.py b/modules/control/controlnetsxs.py index 78890c840..7096902a7 100644 --- a/modules/control/controlnetsxs.py +++ b/modules/control/controlnetsxs.py @@ -81,7 +81,7 @@ class ControlNetXS(): self.model = None self.model_id = None - def load(self, model_id: str = None, time_embedding_mix: float = 0.0): + def load(self, model_id: str = None, time_embedding_mix: float = 0.0) -> str: try: t0 = time.time() model_id = model_id or self.model_id @@ -107,9 +107,11 @@ class ControlNetXS(): t1 = time.time() self.model_id = model_id log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}') + return f'{what} loaded model: {model_id}' except Exception as e: log.error(f'Control {what} model load failed: id="{model_id}" error={e}') errors.display(e, f'Control {what} load') + return f'{what} failed to load model: {model_id}' class ControlNetXSPipeline(): diff --git a/modules/control/processors.py b/modules/control/processors.py index c8cc9438e..ab6196d88 100644 --- a/modules/control/processors.py +++ b/modules/control/processors.py @@ -131,13 +131,13 @@ class Processor(): self.processor_id = None self.override = None - def load(self, processor_id: str = None): + def load(self, processor_id: str = None) -> str: try: t0 = time.time() processor_id = processor_id or self.processor_id if processor_id is None or processor_id == 'None': self.reset() - return + return '' from_config = config.get(processor_id, {}).get('load_config', None) if from_config is not None: for k, v in from_config.items(): @@ -158,7 +158,7 @@ class Processor(): pose_ckpt = 'https://huggingface.co/yzd-v/DWPose/resolve/main/dw-ll_ucoco_384.pth' else: log.error(f'Control processor load failed: id="{processor_id}" error=unknown model type') - return + return f'Processor failed to load: {processor_id}' self.model = cls(det_ckpt=det_ckpt, pose_config=pose_config, pose_ckpt=pose_ckpt, device="cpu") elif 'SegmentAnything' in processor_id: if 'Base' == config['SegmentAnything']['model']: @@ -167,7 +167,7 @@ class Processor(): self.model = cls.from_pretrained(model_path = 'segments-arnaud/sam_vit_l', filename='sam_vit_l_0b3195.pth', model_type='vit_l', **self.load_config) else: log.error(f'Control processor load failed: id="{processor_id}" error=unknown model type') - return + return f'Processor failed to load: {processor_id}' elif config[processor_id].get('load_config', None) is not None: self.model = cls.from_pretrained(**self.load_config) elif config[processor_id]['checkpoint']: @@ -177,9 +177,11 @@ class Processor(): t1 = time.time() self.processor_id = processor_id log.debug(f'Control processor loaded: id="{processor_id}" class={self.model.__class__.__name__} time={t1-t0:.2f}') + return f'Processor loaded: {processor_id}' except Exception as e: log.error(f'Control processor load failed: id="{processor_id}" error={e}') display(e, 'Control processor load') + return f'Processor load filed: {processor_id}' def __call__(self, image_input: Image): if self.override is not None: diff --git a/modules/control/run.py b/modules/control/run.py index 94a5b788f..40868d9a4 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -59,7 +59,7 @@ def restore_pipeline(): devices.torch_gc() -def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bool, +def control_run(units: List[unit.Unit], inputs, inits, unit_type: str, is_generator: bool, input_type: int, prompt, negative, styles, steps, sampler_index, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, full_quality, restore_faces, tiling, @@ -69,7 +69,7 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, ): global pipe, original_pipeline # pylint: disable=global-statement - debug(f'Control {unit_type}: input={inputs}') + debug(f'Control {unit_type}: input={inputs} init={inits} type={input_type}') if inputs is None or (type(inputs) is list and len(inputs) == 0): inputs = [None] output_images: List[Image.Image] = [] # output images @@ -110,7 +110,7 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo hdr_maximize = hdr_maximize, hdr_max_center = hdr_max_center, hdr_max_boundry = hdr_max_boundry, - resize_mode = resize_mode, + resize_mode = resize_mode if resize_name != 'None' else 0, resize_name = resize_name, scale_by = scale_by, selected_scale_tab = selected_scale_tab, @@ -136,36 +136,36 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo if unit_type == 'adapter' and u.adapter.model is not None: active_process.append(u.process) active_model.append(u.adapter) - active_strength.append(u.strength) + active_strength.append(float(u.strength)) p.adapter_conditioning_factor = u.factor shared.log.debug(f'Control T2I-Adapter unit: process={u.process.processor_id} model={u.adapter.model_id} strength={u.strength} factor={u.factor}') elif unit_type == 'controlnet' and u.controlnet.model is not None: active_process.append(u.process) active_model.append(u.controlnet) - active_strength.append(u.strength) - active_start.append(u.start) - active_end.append(u.end) + active_strength.append(float(u.strength)) + active_start.append(float(u.start)) + active_end.append(float(u.end)) p.guess_mode = u.guess shared.log.debug(f'Control ControlNet unit: process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end}') elif unit_type == 'xs' and u.controlnet.model is not None: active_process.append(u.process) active_model.append(u.controlnet) - active_strength.append(u.strength) - active_start.append(u.start) - active_end.append(u.end) + active_strength.append(float(u.strength)) + active_start.append(float(u.start)) + active_end.append(float(u.end)) p.guess_mode = u.guess shared.log.debug(f'Control ControlNet-XS unit: process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end}') elif unit_type == 'reference': p.override = u.override p.attention = u.attention - p.query_weight = u.query_weight - p.adain_weight = u.adain_weight + p.query_weight = float(u.query_weight) + p.adain_weight = float(u.adain_weight) p.fidelity = u.fidelity shared.log.debug('Control Reference unit') else: active_process.append(u.process) # active_model.append(model) - active_strength.append(u.strength) + active_strength.append(float(u.strength)) p.ops.append('control') has_models = False @@ -184,7 +184,6 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo use_conditioning = active_strength[0] if len(active_strength) == 1 else list(active_strength) # strength or list[strength] else: pass - shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset current pipeline debug(f'Control: run type={unit_type} models={has_models}') if unit_type == 'adapter' and has_models: @@ -193,6 +192,8 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo p.task_args['adapter_conditioning_scale'] = use_conditioning instance = adapters.AdapterPipeline(selected_models, shared.sd_model) pipe = instance.pipeline + if inits is not None: + shared.log.warning('Control: T2I-Adapter does not support separate init image') elif unit_type == 'controlnet' and has_models: p.extra_generation_params["Control mode"] = 'ControlNet' p.extra_generation_params["Control conditioning"] = use_conditioning @@ -210,6 +211,8 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo p.control_guidance_end = active_end[0] if len(active_end) == 1 else list(active_end) instance = controlnetsxs.ControlNetXSPipeline(selected_models, shared.sd_model) pipe = instance.pipeline + if inits is not None: + shared.log.warning('Control: ControlNet-XS does not support separate init image') elif unit_type == 'reference': p.extra_generation_params["Control mode"] = 'Reference' p.extra_generation_params["Control attention"] = p.attention @@ -220,6 +223,8 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo p.task_args['style_fidelity'] = p.fidelity instance = reference.ReferencePipeline(shared.sd_model) pipe = instance.pipeline + if inits is not None: + shared.log.warning('Control: ControlNet-XS does not support separate init image') else: # run in img2img mode if len(active_strength) > 0: p.strength = active_strength[0] @@ -244,6 +249,10 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo try: with devices.inference_context(): if isinstance(inputs, str): # only video, the rest is a list + if input_type == 2: # separate init image + if isinstance(inits, str) and inits != inputs: + shared.log.warning('Control: separate init video not support for video input') + input_type = 1 try: video = cv2.VideoCapture(inputs) if not video.isOpened(): @@ -268,6 +277,7 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo if frame is not None: inputs = [Image.fromarray(frame)] # cv2 to pil for i, input_image in enumerate(inputs): + debug(f'Control Control image: {i + 1} of {len(inputs)}') if shared.state.skipped: shared.state.skipped = False continue @@ -278,20 +288,37 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo # get input if isinstance(input_image, str): try: - input_image = Image.open(input_image) + input_image = Image.open(inputs[i]) except Exception as e: - shared.log.error(f'Control: image open failed: path={input_image} error={e}') + shared.log.error(f'Control: image open failed: path={inputs[i]} type=control error={e}') continue + # match init input + if input_type == 1: + debug('Control Init image: same as control') + init_image = input_image + elif inits is None: + debug('Control Init image: none') + init_image = None + elif isinstance(inits[i], str): + debug(f'Control: init image: {inits[i]}') + try: + init_image = Image.open(inits[i]) + except Exception as e: + shared.log.error(f'Control: image open failed: path={inits[i]} type=init error={e}') + continue + else: + debug(f'Control Init image: {i % len(inits) + 1} of {len(inits)}') + init_image = inits[i % len(inits)] index += 1 if video is not None and index % (video_skip_frames + 1) != 0: continue # resize - if resize_mode != 0 and input_image is not None: + if p.resize_mode != 0 and input_image is not None: p.extra_generation_params["Control resize"] = f'{resize_time}: {resize_name}' - if resize_mode != 0 and input_image is not None and resize_time == 'Before': - debug(f'Control resize: image={input_image} width={width} height={height} mode={resize_mode} name={resize_name} sequence={resize_time}') - input_image = images.resize_image(resize_mode, input_image, width, height, resize_name) + if p.resize_mode != 0 and input_image is not None and resize_time == 'Before': + debug(f'Control resize: image={input_image} width={width} height={height} mode={p.resize_mode} name={resize_name} sequence={resize_time}') + input_image = images.resize_image(p.resize_mode, input_image, width, height, resize_name) # process if input_image is None: @@ -335,6 +362,20 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo processed_image = [np.array(i) for i in p.image] processed_image = util.blend(processed_image) # blend all processed images into one processed_image = Image.fromarray(processed_image) + + if unit_type == 'controlnet' and input_type == 1: # Init image same as control + p.task_args['control_image'] = p.image + p.task_args['strength'] = p.denoising_strength + p.task_args['image'] = input_image + elif unit_type == 'controlnet' and input_type == 2: # Separate init image + p.task_args['control_image'] = p.image + p.task_args['strength'] = p.denoising_strength + if init_image is None: + shared.log.warning('Control: separate init image not provided') + p.task_args['image'] = input_image + else: + p.task_args['image'] = init_image + if is_generator: image_txt = f'{processed_image.width}x{processed_image.height}' if processed_image is not None else 'None' msg = f'process | {index} of {frames if video is not None else len(inputs)} | {"Image" if video is None else "Frame"} {image_txt}' @@ -342,23 +383,31 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo yield (None, processed_image, f'Control {msg}') t2 += time.time() - t2 + # prepare pipeline + if hasattr(p, 'init_images'): + del p.init_images # control never uses init_image as-is + if pipe is not None: + if not has_models and (unit_type == 'controlnet' or unit_type == 'adapter' or unit_type == 'xs'): # run in txt2img or img2img mode + if processed_image is not None: + p.init_images = [processed_image] + pipe = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) + else: + pipe = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) + elif unit_type == 'reference': + pipe = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) + else: # actual control + if 'control_image' in p.task_args: + pipe = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) # only controlnet supports img2img + else: + pipe = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) + # pipeline output = None if pipe is not None: # run new pipeline - if not has_models and (unit_type == 'controlnet' or unit_type == 'adapter' or unit_type == 'xs'): # run in img2img mode - if p.image is None: - if hasattr(p, 'init_images'): - del p.init_images - shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset current pipeline - else: - p.init_images = [processed_image] # pylint: disable=attribute-defined-outside-init - shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) # reset current pipeline - else: - if hasattr(p, 'init_images'): - del p.init_images - shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset current pipeline - debug(f'Control exec pipeline: class={pipe.__class__} p={vars(p)}') - debug(f'Control exec pipeline: class={pipe.__class__} args={p.task_args}') + debug(f'Control exec pipeline: class={pipe.__class__}') + debug(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)}') + debug(f'Control exec pipeline: p={vars(p)}') + debug(f'Control exec pipeline: args={p.task_args}') processed: processing.Processed = processing.process_images(p) # run actual pipeline output = processed.images if processed is not None else None # output = pipe(**vars(p)).images # alternative direct pipe exec call @@ -371,9 +420,9 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo output_image = output[0] if output_image is not None: # resize - if resize_mode != 0 and resize_time == 'After': - debug(f'Control resize: image={input_image} width={width} height={height} mode={resize_mode} name={resize_name} sequence={resize_time}') - output_image = images.resize_image(resize_mode, output_image, width, height, resize_name) + if p.resize_mode != 0 and resize_time == 'After': + debug(f'Control resize: image={input_image} width={width} height={height} mode={p.resize_mode} name={resize_name} sequence={resize_time}') + output_image = images.resize_image(p.resize_mode, output_image, width, height, resize_name) elif hasattr(p, 'width') and hasattr(p, 'height'): output_image = output_image.resize((p.width, p.height), Image.Resampling.LANCZOS) diff --git a/modules/control/unit.py b/modules/control/unit.py index 3d52be522..ced17acfd 100644 --- a/modules/control/unit.py +++ b/modules/control/unit.py @@ -32,6 +32,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c image_upload = None, control_start = None, control_end = None, + result_txt = None, extra_controls: list = [], # noqa B006 ): self.enabled = enabled or False @@ -118,17 +119,17 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c # bind ui controls to properties if present if self.type == 'adapter': if model_id is not None: - model_id.change(fn=self.adapter.load, inputs=[model_id], show_progress=True) + model_id.change(fn=self.adapter.load, inputs=[model_id], outputs=[result_txt], show_progress=True) if extra_controls is not None and len(extra_controls) > 0: extra_controls[0].change(fn=adapter_extra, inputs=extra_controls) elif self.type == 'controlnet': if model_id is not None: - model_id.change(fn=self.controlnet.load, inputs=[model_id], show_progress=True) + model_id.change(fn=self.controlnet.load, inputs=[model_id], outputs=[result_txt], show_progress=True) if extra_controls is not None and len(extra_controls) > 0: extra_controls[0].change(fn=controlnet_extra, inputs=extra_controls) elif self.type == 'xs': if model_id is not None: - model_id.change(fn=self.controlnet.load, inputs=[model_id, extra_controls[0]], show_progress=True) + model_id.change(fn=self.controlnet.load, inputs=[model_id, extra_controls[0]], outputs=[result_txt], show_progress=True) if extra_controls is not None and len(extra_controls) > 0: extra_controls[0].change(fn=controlnetxs_extra, inputs=extra_controls) elif self.type == 'reference': @@ -142,7 +143,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c if model_strength is not None: model_strength.change(fn=strength_change, inputs=[model_strength]) if process_id is not None: - process_id.change(fn=self.process.load, inputs=[process_id], show_progress=True) + process_id.change(fn=self.process.load, inputs=[process_id], outputs=[result_txt], show_progress=True) if reset_btn is not None: reset_btn.click(fn=reset, inputs=[], outputs=[enabled_cb, model_id, process_id, model_strength]) if preview_btn is not None: diff --git a/modules/ui.py b/modules/ui.py index f8de15f4e..a47556115 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -208,11 +208,14 @@ def create_advanced_inputs(tab): return cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, full_quality, restore_faces, tiling, hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_center, hdr_max_boundry -def create_resize_inputs(tab, images, time_selector=False, scale_visible=True): +def create_resize_inputs(tab, images, time_selector=False, scale_visible=True, mode=None): dummy_component = gr.Number(visible=False, value=0) with gr.Accordion(open=False, label="Resize", elem_classes=["small-accordion"], elem_id=f"{tab}_resize_group"): with gr.Row(): - resize_mode = gr.Radio(label="Resize mode", elem_id=f"{tab}_resize_mode", choices=shared.resize_modes, type="index", value="None") + if mode is not None: + resize_mode = gr.Radio(label="Resize mode", elem_id=f"{tab}_resize_mode", choices=shared.resize_modes, type="index", value=mode, visible=False) + else: + resize_mode = gr.Radio(label="Resize mode", elem_id=f"{tab}_resize_mode", choices=shared.resize_modes, type="index", value='None') resize_time = gr.Radio(label="Resize order", elem_id=f"{tab}_resize_order", choices=['Before', 'After'], value="Before", visible=time_selector) with gr.Row(): resize_name = gr.Dropdown(label="Resize method", elem_id=f"{tab}_resize_name", choices=[x.name for x in shared.sd_upscalers], value=opts.upscaler_for_img2img) diff --git a/modules/ui_control.py b/modules/ui_control.py index 1fc1deea6..7665843e0 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -14,14 +14,14 @@ gr_height = 512 max_units = 10 units: list[unit.Unit] = [] # main state variable input_source = None -debug = os.environ.get('SD_CONTROL_DEBUG', None) is not None -if debug: - shared.log.trace('Control debug enabled') +input_init = None +debug = shared.log.trace if os.environ.get('SD_CONTROL_DEBUG', None) is not None else lambda *args, **kwargs: None +debug('Trace: CONTROL') def initialize(): from modules import devices - shared.log.debug(f'Control initialize: models={shared.opts.control_dir} debug={debug}') + shared.log.debug(f'Control initialize: models={shared.opts.control_dir}') controlnets.cache_dir = os.path.join(shared.opts.control_dir, 'controlnets') controlnetsxs.cache_dir = os.path.join(shared.opts.control_dir, 'controlnetsxs') adapters.cache_dir = os.path.join(shared.opts.control_dir, 'adapters') @@ -37,8 +37,7 @@ def initialize(): def return_controls(res): # return preview, image, video, gallery, text - if debug: - shared.log.debug(f'Control received: type={type(res)} {res}') + debug(f'Control received: type={type(res)} {res}') if isinstance(res, str): # error response return [None, None, None, None, res] elif isinstance(res, tuple): # standard response received as tuple via control_run->yield(output_images, process_image, result_txt) @@ -67,7 +66,7 @@ def generate_click(job_id: str, active_tab: str, *args): shared.mem_mon.reset() progress.start_task(job_id) try: - for results in control_run(units, input_source, active_tab, True, *args): + for results in control_run(units, input_source, input_init, active_tab, True, *args): progress.record_results(job_id, results) yield return_controls(results) except Exception as e: @@ -106,27 +105,30 @@ def get_video(filepath: str): return msg -def select_input(selected_input): - global input_source # pylint: disable=global-statement +def select_input(selected_input, selected_init, init_type): + debug(f'Control select input: source={selected_input} init={selected_init}, type={init_type}') + global input_source, input_init # pylint: disable=global-statement input_type = type(selected_input) status = 'Control input | Unknown' + res = [gr.Tabs.update(selected='out-gallery'), status] + # control inputs if hasattr(selected_input, 'size'): # image via upload -> image input_source = [selected_input] input_type = 'PIL.Image' shared.log.debug(f'Control input: type={input_type} input={input_source}') status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}' - return [gr.Tabs.update(selected='out-gallery'), status] + res = [gr.Tabs.update(selected='out-gallery'), status] elif isinstance(selected_input, gr.components.image.Image): # not likely input_source = [selected_input.value] input_type = 'gr.Image' shared.log.debug(f'Control input: type={input_type} input={input_source}') - return [gr.Tabs.update(selected='out-gallery'), status] + res = [gr.Tabs.update(selected='out-gallery'), status] elif isinstance(selected_input, str): # video via upload > tmp filepath to video input_source = selected_input input_type = 'gr.Video' shared.log.debug(f'Control input: type={input_type} input={input_source}') status = get_video(input_source) - return [gr.Tabs.update(selected='out-video'), status] + res = [gr.Tabs.update(selected='out-video'), status] elif isinstance(selected_input, list): # batch or folder via upload -> list of tmp filepaths if hasattr(selected_input[0], 'name'): input_type = 'tempfiles' @@ -136,10 +138,46 @@ def select_input(selected_input): input_source = selected_input status = f'Control input | Images | Files {len(input_source)}' shared.log.debug(f'Control input: type={input_type} input={input_source}') - return [gr.Tabs.update(selected='out-gallery'), status] + res = [gr.Tabs.update(selected='out-gallery'), status] else: # unknown input_source = None - return [gr.Tabs.update(selected='out-gallery'), status] + # init inputs: optional + if init_type == 0: # Control only + input_init = None + elif init_type == 1: # Init image same as control assigned during runtime + input_init = None + elif init_type == 2: # Separate init image + if hasattr(selected_init, 'size'): # image via upload -> image + input_init = [selected_init] + input_type = 'PIL.Image' + shared.log.debug(f'Control input: type={input_type} input={input_init}') + status = f'Control input | Image | Size {selected_init.width}x{selected_init.height} | Mode {selected_init.mode}' + res = [gr.Tabs.update(selected='out-gallery'), status] + elif isinstance(selected_init, gr.components.image.Image): # not likely + input_init = [selected_init.value] + input_type = 'gr.Image' + shared.log.debug(f'Control input: type={input_type} input={input_init}') + res = [gr.Tabs.update(selected='out-gallery'), status] + elif isinstance(selected_init, str): # video via upload > tmp filepath to video + input_init = selected_init + input_type = 'gr.Video' + shared.log.debug(f'Control input: type={input_type} input={input_init}') + status = get_video(input_init) + res = [gr.Tabs.update(selected='out-video'), status] + elif isinstance(selected_init, list): # batch or folder via upload -> list of tmp filepaths + if hasattr(selected_init[0], 'name'): + input_type = 'tempfiles' + input_init = [f.name for f in selected_init] # tempfile + else: + input_type = 'files' + input_init = selected_init + status = f'Control input | Images | Files {len(input_init)}' + shared.log.debug(f'Control input: type={input_type} input={input_init}') + res = [gr.Tabs.update(selected='out-gallery'), status] + else: # unknown + input_init = None + debug(f'Control select input: source={input_source} init={input_init}') + return res def video_type_change(video_type): @@ -162,7 +200,13 @@ def create_ui(_blocks: gr.Blocks=None): prompt, styles, negative, btn_generate, _btn_interrogate, _btn_deepbooru, btn_paste, btn_extra, prompt_counter, btn_prompt_counter, negative_counter, btn_negative_counter = ui.create_toprow(is_img2img=False, id_part='control') with FormGroup(elem_id="control_interface", equal_height=False): with gr.Row(elem_id='control_settings'): - resize_mode, resize_name, width, height, scale_by, selected_scale_tab, resize_time = ui.create_resize_inputs('control', [], time_selector=True, scale_visible=False) + + with gr.Accordion(open=False, label="Input", elem_id="control_input", elem_classes=["small-accordion"]): + input_type = gr.Radio(label="Input type", choices=['Control only', 'Init image same as control', 'Separate init image'], value='Control only', type='index', elem_id='control_input_type') + denoising_strength = gr.Slider(minimum=0.01, maximum=0.99, step=0.01, label='Denoising strength', value=0.50, elem_id="control_denoising_strength") + show_preview = gr.Checkbox(label="Show preview", value=True, elem_id="control_show_preview") + + resize_mode, resize_name, width, height, scale_by, selected_scale_tab, resize_time = ui.create_resize_inputs('control', [], time_selector=True, scale_visible=False, mode='Fixed') with gr.Accordion(open=False, label="Sampler", elem_id="control_sampler", elem_classes=["small-accordion"]): sd_samplers.set_samplers() @@ -172,9 +216,6 @@ def create_ui(_blocks: gr.Blocks=None): seed, _reuse_seed, subseed, _reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w = ui.create_seed_inputs('control', reuse_visible=False) cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, full_quality, restore_faces, tiling, hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_center, hdr_max_boundry = ui.create_advanced_inputs('control') - with gr.Accordion(open=False, label="Denoise", elem_id="control_denoise", elem_classes=["small-accordion"]): - denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Denoising strength', value=0.50, elem_id="control_denoising_strength") - with gr.Accordion(open=False, label="Video", elem_id="control_video", elem_classes=["small-accordion"]): with gr.Row(): video_skip_frames = gr.Slider(minimum=0, maximum=100, step=1, label='Skip input frames', value=0, elem_id="control_video_skip_frames") @@ -198,8 +239,8 @@ def create_ui(_blocks: gr.Blocks=None): result_txt = gr.HTML(elem_classes=['control-result'], elem_id='control-result') with gr.Row(elem_id='control-inputs'): - with gr.Column(scale=9, elem_id='control-input-column'): - gr.HTML('Input

') + with gr.Column(scale=9, elem_id='control-input-column', visible=True) as _column_input: + gr.HTML('Control input

') with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-input'): with gr.Tab('Image', id='in-image') as tab_image: input_image = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=True, tool="editor", height=gr_height) @@ -209,7 +250,18 @@ def create_ui(_blocks: gr.Blocks=None): input_batch = gr.File(label="Input", show_label=False, file_count='multiple', file_types=['image'], type='file', interactive=True, height=gr_height) with gr.Tab('Folder', id='in-folder') as tab_folder: input_folder = gr.File(label="Input", show_label=False, file_count='directory', file_types=['image'], type='file', interactive=True, height=gr_height) - with gr.Column(scale=9, elem_id='control-output-column'): + with gr.Column(scale=9, elem_id='control-init-column', visible=False) as column_init: + gr.HTML('Init input

') + with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-init'): + with gr.Tab('Image', id='init-image') as tab_image_init: + init_image = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=True, tool="editor", height=gr_height) + with gr.Tab('Video', id='init-video') as tab_video_init: + init_video = gr.Video(label="Input", show_label=False, interactive=True, height=gr_height) + with gr.Tab('Batch', id='init-batch') as tab_batch_init: + init_batch = gr.File(label="Input", show_label=False, file_count='multiple', file_types=['image'], type='file', interactive=True, height=gr_height) + with gr.Tab('Folder', id='init-folder') as tab_folder_init: + init_folder = gr.File(label="Input", show_label=False, file_count='directory', file_types=['image'], type='file', interactive=True, height=gr_height) + with gr.Column(scale=9, elem_id='control-output-column', visible=True) as _column_output: gr.HTML('Output

') with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-output') as output_tabs: with gr.Tab('Gallery', id='out-gallery'): @@ -218,20 +270,21 @@ def create_ui(_blocks: gr.Blocks=None): output_image = gr.Image(label="Input", show_label=False, type="pil", interactive=False, tool="editor", height=gr_height) with gr.Tab('Video', id='out-video'): output_video = gr.Video(label="Input", show_label=False, height=gr_height) - with gr.Column(scale=9, elem_id='control-preview-column'): + with gr.Column(scale=9, elem_id='control-preview-column', visible=True) as column_preview: gr.HTML('Preview

') with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-preview'): with gr.Tab('Preview', id='preview-image') as tab_image: preview_process = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=False, height=gr_height, visible=True) - input_image.change(fn=select_input, inputs=[input_image], outputs=[output_tabs, result_txt]) - input_video.change(fn=select_input, inputs=[input_video], outputs=[output_tabs, result_txt]) - input_batch.change(fn=select_input, inputs=[input_batch], outputs=[output_tabs, result_txt]) - input_folder.change(fn=select_input, inputs=[input_folder], outputs=[output_tabs, result_txt]) - tab_image.select(fn=select_input, inputs=[input_image], outputs=[output_tabs, result_txt]) - tab_video.select(fn=select_input, inputs=[input_video], outputs=[output_tabs, result_txt]) - tab_batch.select(fn=select_input, inputs=[input_batch], outputs=[output_tabs, result_txt]) - tab_folder.select(fn=select_input, inputs=[input_folder], outputs=[output_tabs, result_txt]) + for ctrl in [input_image, input_video, input_batch, input_folder, init_image, init_video, init_batch, init_folder, tab_image, tab_video, tab_batch, tab_folder, tab_image_init, tab_video_init, tab_batch_init, tab_folder_init]: + inputs = [input_image, init_image, input_type] + outputs = [output_tabs, result_txt] + if hasattr(ctrl, 'change'): + ctrl.change(fn=select_input, inputs=inputs, outputs=outputs) + if hasattr(ctrl, 'select'): + ctrl.select(fn=select_input, inputs=inputs, outputs=outputs) + show_preview.change(fn=lambda x: gr.update(visible=x), inputs=[show_preview], outputs=[column_preview]) + input_type.change(fn=lambda x: gr.update(visible=x == 2), inputs=[input_type], outputs=[column_init]) with gr.Tabs(elem_id='control-tabs') as _tabs_control_type: @@ -252,15 +305,16 @@ def create_ui(_blocks: gr.Blocks=None): process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') model_id = gr.Dropdown(label="ControlNet", choices=controlnets.list_models(), value='None') ui_common.create_refresh_button(model_id, controlnets.list_models, lambda: {"choices": controlnets.list_models(refresh=True)}, 'refresh_control_models') - model_strength = gr.Slider(label="Strength", minimum=0.0, maximum=1.0, step=0.1, value=1.0-i/10) - control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.1, value=0) - control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.1, value=1.0) + model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) + control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0) + control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0) reset_btn = ui_components.ToolButton(value=ui_symbols.reset) image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) process_btn= ui_components.ToolButton(value=ui_symbols.preview) controlnet_ui_units.append(unit_ui) units.append(unit.Unit( unit_type = 'controlnet', + result_txt = result_txt, image_input = input_image, enabled_cb = enabled_cb, reset_btn = reset_btn, @@ -296,15 +350,16 @@ def create_ui(_blocks: gr.Blocks=None): process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') model_id = gr.Dropdown(label="ControlNet-XS", choices=controlnetsxs.list_models(), value='None') ui_common.create_refresh_button(model_id, controlnetsxs.list_models, lambda: {"choices": controlnetsxs.list_models(refresh=True)}, 'refresh_control_models') - model_strength = gr.Slider(label="Strength", minimum=0.0, maximum=1.0, step=0.1, value=1.0-i/10) - control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.1, value=0) - control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.1, value=1.0) + model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) + control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0) + control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0) reset_btn = ui_components.ToolButton(value=ui_symbols.reset) image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) process_btn= ui_components.ToolButton(value=ui_symbols.preview) controlnetxs_ui_units.append(unit_ui) units.append(unit.Unit( unit_type = 'xs', + result_txt = result_txt, image_input = input_image, enabled_cb = enabled_cb, reset_btn = reset_btn, @@ -327,7 +382,7 @@ def create_ui(_blocks: gr.Blocks=None): gr.HTML('T2I-Adapter') with gr.Row(): extra_controls = [ - gr.Slider(label="Control factor", minimum=0.0, maximum=1.0, step=0.1, value=1.0, scale=3), + gr.Slider(label="Control factor", minimum=0.0, maximum=1.0, step=0.05, value=1.0, scale=3), ] num_adaptor_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) adaptor_ui_units = [] # list of hidable accordions @@ -340,13 +395,14 @@ def create_ui(_blocks: gr.Blocks=None): process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') model_id = gr.Dropdown(label="Adapter", choices=adapters.list_models(), value='None') ui_common.create_refresh_button(model_id, adapters.list_models, lambda: {"choices": adapters.list_models(refresh=True)}, 'refresh_adapter_models') - model_strength = gr.Slider(label="Strength", minimum=0.0, maximum=1.0, step=0.1, value=1.0-i/10) + model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) reset_btn = ui_components.ToolButton(value=ui_symbols.reset) image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) process_btn= ui_components.ToolButton(value=ui_symbols.preview) adaptor_ui_units.append(unit_ui) units.append(unit.Unit( unit_type = 'adapter', + result_txt = result_txt, image_input = input_image, enabled_cb = enabled_cb, reset_btn = reset_btn, @@ -368,9 +424,9 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Row(): extra_controls = [ gr.Radio(label="Reference context", choices=['Attention', 'Adain', 'Attention Adain'], value='Attention', interactive=True), - gr.Slider(label="Style fidelity", minimum=0.0, maximum=1.0, step=0.1, value=0.5, interactive=True), # prompt vs control importance - gr.Slider(label="Reference query weight", minimum=0.0, maximum=1.0, step=0.1, value=1.0, interactive=True), - gr.Slider(label="Reference adain weight", minimum=0.0, maximum=2.0, step=0.1, value=1.0, interactive=True), + gr.Slider(label="Style fidelity", minimum=0.0, maximum=1.0, step=0.05, value=0.5, interactive=True), # prompt vs control importance + gr.Slider(label="Reference query weight", minimum=0.0, maximum=1.0, step=0.05, value=1.0, interactive=True), + gr.Slider(label="Reference adain weight", minimum=0.0, maximum=2.0, step=0.05, value=1.0, interactive=True), ] for i in range(1): # can only have one reference unit with gr.Accordion(f'Reference unit {i+1}', visible=True) as unit_ui: @@ -379,12 +435,13 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Row(): enabled_cb = gr.Checkbox(value= i == 0, label="Enabled", visible=False) model_id = gr.Dropdown(label="Reference", choices=reference.list_models(), value='Reference', visible=False) - model_strength = gr.Slider(label="Strength", minimum=0.0, maximum=1.0, step=0.1, value=1.0, visible=False) + model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, visible=False) reset_btn = ui_components.ToolButton(value=ui_symbols.reset) image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) process_btn= ui_components.ToolButton(value=ui_symbols.preview) units.append(unit.Unit( unit_type = 'reference', + result_txt = result_txt, image_input = input_image, enabled_cb = enabled_cb, reset_btn = reset_btn, @@ -443,6 +500,7 @@ def create_ui(_blocks: gr.Blocks=None): tabs_state = gr.Text(value='none', visible=False) input_fields = [ + input_type, prompt, negative, styles, steps, sampler_index, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, @@ -476,7 +534,7 @@ def create_ui(_blocks: gr.Blocks=None): bindings = generation_parameters_copypaste.ParamBinding(paste_button=btn_paste, tabname="control", source_text_component=prompt, source_image_component=output_gallery) generation_parameters_copypaste.register_paste_params_button(bindings) - if debug: + if os.environ.get('SD_CONTROL_DEBUG', None) is not None: # debug only from modules.control.test import test_processors, test_controlnets, test_adapters, test_xs gr.HTML('

Debug


') with gr.Row(): diff --git a/wiki b/wiki index f75201d5b..9994450f3 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit f75201d5bc7a12c176e7243845c8d96835ef726a +Subproject commit 9994450f36413a1a72500c397ff39e024aab13a1