diff --git a/CHANGELOG.md b/CHANGELOG.md index 51b9ebe96..89541a311 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,11 +12,11 @@ OPTIONAL: - mixture tiling [pr](https://github.com/huggingface/diffusers/tree/main/examples/community#stable-diffusion-mixture-tiling) - instaflow [pr](https://github.com/huggingface/diffusers/pull/6057)[repo](https://github.com/gnobitab/RectifiedFlow) - control api -- face api +- control params paste - masking api - preprocess api -## Update for 2023-01-29 +## Update for 2023-01-30 Another big release, highlights being: - A lot more functionality in the **Control** module: diff --git a/modules/control/run.py b/modules/control/run.py index 8b443be3d..7132453fe 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -198,6 +198,9 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ has_models = False selected_models: List[Union[controlnet.ControlNetModel, xs.ControlNetXSModel, t2iadapter.AdapterModel]] = None + control_conditioning = None + control_guidance_start = None + control_guidance_end = None if unit_type == 'adapter' or unit_type == 'controlnet' or unit_type == 'xs' or unit_type == 'lite': if len(active_model) == 0: selected_models = None @@ -205,46 +208,48 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ selected_models = active_model[0].model if active_model[0].model is not None else None p.extra_generation_params["Control model"] = (active_model[0].model_id or '') if active_model[0].model is not None else None has_models = selected_models is not None + control_conditioning = active_strength[0] + control_guidance_start = active_start[0] + control_guidance_end = active_end[0] else: selected_models = [m.model for m in active_model if m.model is not None] p.extra_generation_params["Control model"] = ', '.join([(m.model_id or '') for m in active_model if m.model is not None]) has_models = len(selected_models) > 0 - use_conditioning = active_strength[0] if len(active_strength) == 1 else list(active_strength) # strength or list[strength] + control_conditioning = active_strength[0] if len(active_strength) == 1 else list(active_strength) # strength or list[strength] + control_guidance_start = active_start[0] if len(active_start) == 1 else list(active_start) + control_guidance_end = active_end[0] if len(active_end) == 1 else list(active_end) + p.extra_generation_params["Control conditioning"] = control_conditioning else: pass debug(f'Control: run type={unit_type} models={has_models}') if unit_type == 'adapter' and has_models: p.extra_generation_params["Control mode"] = 'T2I-Adapter' - p.extra_generation_params["Control conditioning"] = use_conditioning - p.task_args['adapter_conditioning_scale'] = use_conditioning + p.task_args['adapter_conditioning_scale'] = control_conditioning instance = t2iadapter.AdapterPipeline(selected_models, shared.sd_model) pipe = instance.pipeline if inits is not None: shared.log.warning('Control: T2I-Adapter does not support separate init image') elif unit_type == 'controlnet' and has_models: p.extra_generation_params["Control mode"] = 'ControlNet' - p.extra_generation_params["Control conditioning"] = use_conditioning - p.task_args['controlnet_conditioning_scale'] = use_conditioning - p.task_args['control_guidance_start'] = active_start[0] if len(active_start) == 1 else list(active_start) - p.task_args['control_guidance_end'] = active_end[0] if len(active_end) == 1 else list(active_end) + p.task_args['controlnet_conditioning_scale'] = control_conditioning + p.task_args['control_guidance_start'] = control_guidance_start + p.task_args['control_guidance_end'] = control_guidance_end p.task_args['guess_mode'] = p.guess_mode instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model) pipe = instance.pipeline elif unit_type == 'xs' and has_models: p.extra_generation_params["Control mode"] = 'ControlNet-XS' - p.extra_generation_params["Control conditioning"] = use_conditioning - p.controlnet_conditioning_scale = use_conditioning - p.control_guidance_start = active_start[0] if len(active_start) == 1 else list(active_start) - p.control_guidance_end = active_end[0] if len(active_end) == 1 else list(active_end) + p.controlnet_conditioning_scale = control_conditioning + p.control_guidance_start = control_guidance_start + p.control_guidance_end = control_guidance_end instance = xs.ControlNetXSPipeline(selected_models, shared.sd_model) pipe = instance.pipeline if inits is not None: shared.log.warning('Control: ControlNet-XS does not support separate init image') elif unit_type == 'lite' and has_models: p.extra_generation_params["Control mode"] = 'ControlLLLite' - p.extra_generation_params["Control conditioning"] = use_conditioning - p.controlnet_conditioning_scale = use_conditioning + p.controlnet_conditioning_scale = control_conditioning instance = lite.ControlLLitePipeline(shared.sd_model) pipe = instance.pipeline if inits is not None: @@ -376,7 +381,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ p.height = input_image.height debug(f'Control: input image={input_image}') - p.image = [] + processed_images = [] masked_image = masking.run_mask(input_image=input_image, input_mask=mask, return_type='Masked') if mask is not None else input_image for i, process in enumerate(active_process): # list[image] image_mode = 'L' if unit_type == 'adapter' and len(active_model) > i and ('Canny' in active_model[i].model_id or 'Sketch' in active_model[i].model_id) else 'RGB' # t2iadapter canny and sketch work in grayscale only @@ -390,23 +395,35 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ scale_by=scale_by_before, ) if processed_image is not None: - p.image.append(processed_image) + processed_images.append(processed_image) if shared.opts.control_unload_processor: processors.config[process.processor_id]['dirty'] = True # to force reload process.model = None - if len(p.image) > 0: + debug(f'Control processed: {len(processed_images)}') + if len(processed_images) > 0: p.extra_generation_params["Control process"] = [p.processor_id for p in active_process] - if any(img is None for img in p.image): + if any(img is None for img in processed_images): msg = 'Control: attempting process but output is none' - shared.log.error(f'{msg}: {p.image}') + shared.log.error(f'{msg}: {processed_images}') restore_pipeline() return msg - p.init_images = p.image - processed_image = [np.array(i) for i in p.image] + processed_image = [np.array(i) for i in processed_images] processed_image = util.blend(processed_image) # blend all processed images into one processed_image = Image.fromarray(processed_image) + if isinstance(selected_models, list) and len(processed_images) == len(selected_models): + debug(f'Control: inputs match: input={len(processed_images)} models={len(selected_models)}') + p.init_images = processed_images + elif isinstance(selected_models, list) and len(processed_images) != len(selected_models): + msg = f'Control: number of inputs does not match: input={len(processed_images)} models={len(selected_models)}' + shared.log.error(msg) + restore_pipeline() + return msg + elif selected_models is not None: + debug('Control: single model - blending images') + p.init_images = [processed_image] else: + debug('Control processed: using input direct') processed_image = input_image if unit_type == 'reference': @@ -469,7 +486,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ if hasattr(p, 'init_images') and p.init_images is not None: p.task_args['image'] = p.init_images # need to set explicitly for txt2img if unit_type == 'lite': - instance.apply(selected_models, p.image, use_conditioning) + instance.apply(selected_models, p.image, control_conditioning) if hasattr(p, 'init_images') and p.init_images is None: del p.init_images diff --git a/modules/control/unit.py b/modules/control/unit.py index 1b49f456c..de7decdd4 100644 --- a/modules/control/unit.py +++ b/modules/control/unit.py @@ -37,7 +37,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c result_txt = None, extra_controls: list = [], # noqa B006 ): - self.enabled = enabled or True + self.enabled = enabled or False self.type = unit_type self.strength = strength or 1.0 self.start = start or 0 diff --git a/modules/ui_control.py b/modules/ui_control.py index 7b9905332..ac49964e2 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -398,9 +398,10 @@ def create_ui(_blocks: gr.Blocks=None): num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) controlnet_ui_units = [] # list of hidable accordions for i in range(max_units): + enabled = True if i==0 else False with gr.Accordion(f'ControlNet unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: with gr.Row(): - enabled_cb = gr.Checkbox(value=True, label="") + enabled_cb = gr.Checkbox(enabled, container=False, show_label=False) process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None') ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, f'refresh_controlnet_models_{i}') @@ -414,6 +415,7 @@ def create_ui(_blocks: gr.Blocks=None): controlnet_ui_units.append(unit_ui) units.append(unit.Unit( unit_type = 'controlnet', + enabled = enabled, result_txt = result_txt, enabled_cb = enabled_cb, reset_btn = reset_btn, @@ -451,9 +453,10 @@ def create_ui(_blocks: gr.Blocks=None): num_adapter_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) adapter_ui_units = [] # list of hidable accordions for i in range(max_units): + enabled = True if i==0 else False with gr.Accordion(f'T2I-Adapter unit {i+1}', visible= i < num_adapter_units.value, elem_classes='control-unit') as unit_ui: with gr.Row(): - enabled_cb = gr.Checkbox(value=True, label="Enabled") + enabled_cb = gr.Checkbox(enabled, container=False, show_label=False) process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') model_id = gr.Dropdown(label="Adapter", choices=t2iadapter.list_models(), value='None') ui_common.create_refresh_button(model_id, t2iadapter.list_models, lambda: {"choices": t2iadapter.list_models(refresh=True)}, f'refresh_adapter_models_{i}') @@ -465,6 +468,7 @@ def create_ui(_blocks: gr.Blocks=None): adapter_ui_units.append(unit_ui) units.append(unit.Unit( unit_type = 'adapter', + enabled = enabled, result_txt = result_txt, enabled_cb = enabled_cb, reset_btn = reset_btn, @@ -491,9 +495,10 @@ def create_ui(_blocks: gr.Blocks=None): num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) controlnetxs_ui_units = [] # list of hidable accordions for i in range(max_units): + enabled = True if i==0 else False with gr.Accordion(f'ControlNet-XS unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: with gr.Row(): - enabled_cb = gr.Checkbox(value=True, label="") + enabled_cb = gr.Checkbox(enabled, container=False, show_label=False) process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') model_id = gr.Dropdown(label="ControlNet-XS", choices=xs.list_models(), value='None') ui_common.create_refresh_button(model_id, xs.list_models, lambda: {"choices": xs.list_models(refresh=True)}, f'refresh_xs_models_{i}') @@ -507,6 +512,7 @@ def create_ui(_blocks: gr.Blocks=None): controlnetxs_ui_units.append(unit_ui) units.append(unit.Unit( unit_type = 'xs', + enabled = enabled, result_txt = result_txt, enabled_cb = enabled_cb, reset_btn = reset_btn, @@ -534,9 +540,10 @@ def create_ui(_blocks: gr.Blocks=None): num_lite_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) lite_ui_units = [] # list of hidable accordions for i in range(max_units): + enabled = True if i==0 else False with gr.Accordion(f'Control-LLLite unit {i+1}', visible= i < num_lite_units.value, elem_classes='control-unit') as unit_ui: with gr.Row(): - enabled_cb = gr.Checkbox(value=True, label="Enabled") + enabled_cb = gr.Checkbox(enabled, container=False, show_label=False) process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') model_id = gr.Dropdown(label="Model", choices=lite.list_models(), value='None') ui_common.create_refresh_button(model_id, lite.list_models, lambda: {"choices": lite.list_models(refresh=True)}, f'refresh_lite_models_{i}') @@ -548,6 +555,7 @@ def create_ui(_blocks: gr.Blocks=None): lite_ui_units.append(unit_ui) units.append(unit.Unit( unit_type = 'lite', + enabled = enabled, result_txt = result_txt, enabled_cb = enabled_cb, reset_btn = reset_btn, @@ -575,9 +583,10 @@ def create_ui(_blocks: gr.Blocks=None): gr.Slider(label="Reference adain weight", minimum=0.0, maximum=2.0, step=0.05, value=1.0, interactive=True), ] for i in range(1): # can only have one reference unit + enabled = True if i==0 else False with gr.Accordion(f'Reference unit {i+1}', visible=True, elem_classes='control-unit') as unit_ui: with gr.Row(): - enabled_cb = gr.Checkbox(value=True, label="Enabled", visible=False) + enabled_cb = gr.Checkbox(enabled, container=False, show_label=False) model_id = gr.Dropdown(label="Reference", choices=reference.list_models(), value='Reference', visible=False) model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, visible=False) reset_btn = ui_components.ToolButton(value=ui_symbols.reset) @@ -586,6 +595,7 @@ def create_ui(_blocks: gr.Blocks=None): process_btn= ui_components.ToolButton(value=ui_symbols.preview) units.append(unit.Unit( unit_type = 'reference', + enabled = enabled, result_txt = result_txt, enabled_cb = enabled_cb, reset_btn = reset_btn,