diff --git a/CHANGELOG.md b/CHANGELOG.md index a26efb7be..b178ab845 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2023-01-05 +## Update for 2023-01-06 Following-up on a major release, some more functionality in new Control module And it also includes fixes for all reported issues so far @@ -21,6 +21,8 @@ And it also includes fixes for all reported issues so far - add **openpose xl** controlnet - configurable output folder in settings - auto-refresh available models on tab activate + - add image preview for override images set per-unit + - more compact unit layout - reduce usage of temp files - add context menu to action buttons - resize by now applies to input image or frame individually diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 9d1f0a07f..94c93c670 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 9d1f0a07fa2754338a6b260fad2abaf9d3f42ade +Subproject commit 94c93c6708394cfb56bea50772388e3a65e78a01 diff --git a/javascript/sdnext.css b/javascript/sdnext.css index fb3a3ccb7..194b44447 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -261,7 +261,9 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt #control_input_type { max-width: 18em } #control_settings .small-accordion .form { min-width: 350px !important } .control-button { min-height: 42px; max-height: 42px; line-height: 1em; } -.control-tabs>.tab-nav { margin-bottom: 0; margin-top: 0; } +.control-tabs > .tab-nav { margin-bottom: 0; margin-top: 0; } +.control-unit { max-width: 1200px; padding: 0 !important; margin-top: -10px !important; } +.control-unit > .label-wrap { margin-bottom: 0 !important; } .processor-settings { padding: 0 !important; max-width: 300px; } .processor-group>div { flex-flow: wrap;gap: 1em; } diff --git a/modules/control/run.py b/modules/control/run.py index 056090494..742cde298 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -467,11 +467,9 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ # pipeline output = None if pipe is not None: # run new pipeline - debug(f'Control exec pipeline: class={pipe.__class__}') - debug(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)}') + debug(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)} class={pipe.__class__}') debug(f'Control exec pipeline: p={vars(p)}') - debug(f'Control exec pipeline: args={p.task_args}') - debug(f'Control exec pipeline: image={p.task_args.get("image", None)} control={p.task_args.get("control_image", None)} mask={p.task_args.get("mask_image", None)} ref={p.task_args.get("ref_image", None)}') + debug(f'Control exec pipeline: args={p.task_args} image={p.task_args.get("image", None)} control={p.task_args.get("control_image", None)} mask={p.task_args.get("mask_image", None)} ref={p.task_args.get("ref_image", None)}') processed: processing.Processed = processing.process_images(p) # run actual pipeline output = processed.images if processed is not None else None # output = pipe(**vars(p)).images # alternative direct pipe exec call diff --git a/modules/control/unit.py b/modules/control/unit.py index 1df54b81b..0865aba01 100644 --- a/modules/control/unit.py +++ b/modules/control/unit.py @@ -1,5 +1,6 @@ from typing import Union from PIL import Image +import gradio as gr from modules.shared import log from modules.control import processors from modules.control.units import controlnet @@ -31,6 +32,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c image_input = None, preview_process = None, image_upload = None, + image_preview = None, control_start = None, control_end = None, result_txt = None, @@ -101,8 +103,10 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c self.process.override = Image.open(image_file.name) self.override = self.process.override log.debug(f'Control process upload image: path="{image_file.name}" image={self.process.override}') + return gr.update(visible=self.process.override is not None, value=self.process.override) except Exception as e: log.error(f'Control process upload image failed: path="{image_file.name}" error={e}') + return gr.update(visible=False, value=None) # actual init if self.type == 'adapter': @@ -157,7 +161,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c if preview_btn is not None: preview_btn.click(fn=self.process.preview, inputs=[self.input], outputs=[preview_process]) # return list of images for gallery if image_upload is not None: - image_upload.upload(fn=upload_image, inputs=[image_upload], outputs=[]) # return list of images for gallery + image_upload.upload(fn=upload_image, inputs=[image_upload], outputs=[image_preview]) # return list of images for gallery if control_start is not None and control_end is not None: control_start.change(fn=control_change, inputs=[control_start, control_end]) control_end.change(fn=control_change, inputs=[control_start, control_end]) diff --git a/modules/control/units/reference.py b/modules/control/units/reference.py index 8d07ecceb..d7f0fb104 100644 --- a/modules/control/units/reference.py +++ b/modules/control/units/reference.py @@ -3,7 +3,7 @@ import time from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline from modules.control.proc.reference_sd15 import StableDiffusionReferencePipeline from modules.control.proc.reference_sdxl import StableDiffusionXLReferencePipeline -from modules.shared import log +from modules.shared import log, opts from modules.control.units import detect @@ -22,6 +22,8 @@ class ReferencePipeline(): if pipeline is None: log.error(f'Control {what} model pipeline: model not loaded') return + if opts.diffusers_fuse_projections and hasattr(pipeline, 'unfuse_qkv_projections'): + pipeline.unfuse_qkv_projections() if detect.is_sdxl(pipeline): self.pipeline = StableDiffusionXLReferencePipeline( vae=pipeline.vae, diff --git a/modules/sd_models.py b/modules/sd_models.py index 4267f4cbc..51a3b4adc 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1036,6 +1036,10 @@ def set_diffuser_pipe(pipe, new_pipe_type): image_encoder = getattr(pipe, "image_encoder", None) feature_extractor = getattr(pipe, "feature_extractor", None) + # skip specific pipelines + if pipe.__class__.__name__ == 'StableDiffusionReferencePipeline' or pipe.__class__.__name__ == 'StableDiffusionAdapterPipeline': + return pipe + try: if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: new_pipe = diffusers.AutoPipelineForText2Image.from_pipe(pipe) @@ -1044,7 +1048,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): elif new_pipe_type == DiffusersTaskType.INPAINTING: new_pipe = diffusers.AutoPipelineForInpainting.from_pipe(pipe) except Exception as e: # pylint: disable=unused-variable - shared.log.error(f'Failed to change: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}') + shared.log.warning(f'Failed to change: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}') return pipe if pipe.__class__ == new_pipe.__class__: @@ -1263,10 +1267,10 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model') def convert_to_faketensors(tensor): - fake_module = torch._subclasses.fake_tensor.FakeTensorMode(allow_non_fake_inputs=True) - if hasattr(tensor, "weight"): - tensor.weight = torch.nn.Parameter(fake_module.from_tensor(tensor.weight)) - return tensor + fake_module = torch._subclasses.fake_tensor.FakeTensorMode(allow_non_fake_inputs=True) # pylint: disable=protected-access + if hasattr(tensor, "weight"): + tensor.weight = torch.nn.Parameter(fake_module.from_tensor(tensor.weight)) + return tensor def disable_offload(sd_model): diff --git a/modules/ui_control.py b/modules/ui_control.py index eb8a4889c..9642340af 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -421,20 +421,19 @@ def create_ui(_blocks: gr.Blocks=None): num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) controlnet_ui_units = [] # list of hidable accordions for i in range(max_units): - with gr.Accordion(f'Control unit {i+1}', visible= i < num_controlnet_units.value) as unit_ui: + with gr.Accordion(f'Control unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: with gr.Row(): - with gr.Column(): - with gr.Row(): - enabled_cb = gr.Checkbox(value= i==0, label="") - process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') - model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None') - ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, 'refresh_controlnet_models') - model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) - control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0) - control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0) - reset_btn = ui_components.ToolButton(value=ui_symbols.reset) - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) - process_btn= ui_components.ToolButton(value=ui_symbols.preview) + enabled_cb = gr.Checkbox(value= i==0, label="") + process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') + model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None') + ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, 'refresh_controlnet_models') + model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) + control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0) + control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0) + reset_btn = ui_components.ToolButton(value=ui_symbols.reset) + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) + process_btn= ui_components.ToolButton(value=ui_symbols.preview) + image_preview = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=False, height=128, width=128, visible=False) controlnet_ui_units.append(unit_ui) units.append(unit.Unit( unit_type = 'controlnet', @@ -448,6 +447,7 @@ def create_ui(_blocks: gr.Blocks=None): preview_process = preview_process, preview_btn = process_btn, image_upload = image_upload, + image_preview = image_preview, control_start = control_start, control_end = control_end, extra_controls = extra_controls, @@ -457,51 +457,6 @@ def create_ui(_blocks: gr.Blocks=None): units[-1].enabled = True # enable first unit in group num_controlnet_units.change(fn=display_units, inputs=[num_controlnet_units], outputs=controlnet_ui_units) - with gr.Tab('XS') as _tab_controlnetxs: - gr.HTML('ControlNet XS') - with gr.Row(): - extra_controls = [ - gr.Slider(label="Time embedding mix", minimum=0.0, maximum=1.0, step=0.05, value=0.0, scale=3) - ] - num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) - controlnetxs_ui_units = [] # list of hidable accordions - for i in range(max_units): - with gr.Accordion(f'Control unit {i+1}', visible= i < num_controlnet_units.value) as unit_ui: - with gr.Row(): - with gr.Column(): - with gr.Row(): - enabled_cb = gr.Checkbox(value= i==0, label="") - process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') - model_id = gr.Dropdown(label="ControlNet-XS", choices=xs.list_models(), value='None') - ui_common.create_refresh_button(model_id, xs.list_models, lambda: {"choices": xs.list_models(refresh=True)}, 'refresh_xs_models') - model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) - control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0) - control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0) - reset_btn = ui_components.ToolButton(value=ui_symbols.reset) - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) - process_btn= ui_components.ToolButton(value=ui_symbols.preview) - controlnetxs_ui_units.append(unit_ui) - units.append(unit.Unit( - unit_type = 'xs', - result_txt = result_txt, - image_input = input_image, - enabled_cb = enabled_cb, - reset_btn = reset_btn, - process_id = process_id, - model_id = model_id, - model_strength = model_strength, - preview_process = preview_process, - preview_btn = process_btn, - image_upload = image_upload, - control_start = control_start, - control_end = control_end, - extra_controls = extra_controls, - ) - ) - if i == 0: - units[-1].enabled = True # enable first unit in group - num_controlnet_units.change(fn=display_units, inputs=[num_controlnet_units], outputs=controlnetxs_ui_units) - with gr.Tab('Adapter') as _tab_adapter: gr.HTML('T2I-Adapter') with gr.Row(): @@ -511,18 +466,17 @@ def create_ui(_blocks: gr.Blocks=None): num_adapter_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) adapter_ui_units = [] # list of hidable accordions for i in range(max_units): - with gr.Accordion(f'Adapter unit {i+1}', visible= i < num_adapter_units.value) as unit_ui: + with gr.Accordion(f'Adapter unit {i+1}', visible= i < num_adapter_units.value, elem_classes='control-unit') as unit_ui: with gr.Row(): - with gr.Column(): - with gr.Row(): - enabled_cb = gr.Checkbox(value= i == 0, label="Enabled") - process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') - model_id = gr.Dropdown(label="Adapter", choices=t2iadapter.list_models(), value='None') - ui_common.create_refresh_button(model_id, t2iadapter.list_models, lambda: {"choices": t2iadapter.list_models(refresh=True)}, 'refresh_adapter_models') - model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) - reset_btn = ui_components.ToolButton(value=ui_symbols.reset) - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) - process_btn= ui_components.ToolButton(value=ui_symbols.preview) + enabled_cb = gr.Checkbox(value= i == 0, label="Enabled") + process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') + model_id = gr.Dropdown(label="Adapter", choices=t2iadapter.list_models(), value='None') + ui_common.create_refresh_button(model_id, t2iadapter.list_models, lambda: {"choices": t2iadapter.list_models(refresh=True)}, 'refresh_adapter_models') + model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) + reset_btn = ui_components.ToolButton(value=ui_symbols.reset) + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) + process_btn= ui_components.ToolButton(value=ui_symbols.preview) + image_preview = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=False, height=128, width=128, visible=False) adapter_ui_units.append(unit_ui) units.append(unit.Unit( unit_type = 'adapter', @@ -536,6 +490,7 @@ def create_ui(_blocks: gr.Blocks=None): preview_process = preview_process, preview_btn = process_btn, image_upload = image_upload, + image_preview = image_preview, extra_controls = extra_controls, ) ) @@ -543,6 +498,51 @@ def create_ui(_blocks: gr.Blocks=None): units[-1].enabled = True # enable first unit in group num_adapter_units.change(fn=display_units, inputs=[num_adapter_units], outputs=adapter_ui_units) + with gr.Tab('XS') as _tab_controlnetxs: + gr.HTML('ControlNet XS') + with gr.Row(): + extra_controls = [ + gr.Slider(label="Time embedding mix", minimum=0.0, maximum=1.0, step=0.05, value=0.0, scale=3) + ] + num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) + controlnetxs_ui_units = [] # list of hidable accordions + for i in range(max_units): + with gr.Accordion(f'Control unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: + with gr.Row(): + enabled_cb = gr.Checkbox(value= i==0, label="") + process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') + model_id = gr.Dropdown(label="ControlNet-XS", choices=xs.list_models(), value='None') + ui_common.create_refresh_button(model_id, xs.list_models, lambda: {"choices": xs.list_models(refresh=True)}, 'refresh_xs_models') + model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) + control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0) + control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0) + reset_btn = ui_components.ToolButton(value=ui_symbols.reset) + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) + process_btn= ui_components.ToolButton(value=ui_symbols.preview) + image_preview = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=False, height=128, width=128, visible=False) + controlnetxs_ui_units.append(unit_ui) + units.append(unit.Unit( + unit_type = 'xs', + result_txt = result_txt, + image_input = input_image, + enabled_cb = enabled_cb, + reset_btn = reset_btn, + process_id = process_id, + model_id = model_id, + model_strength = model_strength, + preview_process = preview_process, + preview_btn = process_btn, + image_upload = image_upload, + image_preview = image_preview, + control_start = control_start, + control_end = control_end, + extra_controls = extra_controls, + ) + ) + if i == 0: + units[-1].enabled = True # enable first unit in group + num_controlnet_units.change(fn=display_units, inputs=[num_controlnet_units], outputs=controlnetxs_ui_units) + with gr.Tab('Lite') as _tab_lite: gr.HTML('Control LLLite') with gr.Row(): @@ -551,18 +551,17 @@ def create_ui(_blocks: gr.Blocks=None): num_lite_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) lite_ui_units = [] # list of hidable accordions for i in range(max_units): - with gr.Accordion(f'Control unit {i+1}', visible= i < num_lite_units.value) as unit_ui: + with gr.Accordion(f'Control unit {i+1}', visible= i < num_lite_units.value, elem_classes='control-unit') as unit_ui: with gr.Row(): - with gr.Column(): - with gr.Row(): - enabled_cb = gr.Checkbox(value= i == 0, label="Enabled") - process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') - model_id = gr.Dropdown(label="Model", choices=lite.list_models(), value='None') - ui_common.create_refresh_button(model_id, lite.list_models, lambda: {"choices": lite.list_models(refresh=True)}, 'refresh_lite_models') - model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) - reset_btn = ui_components.ToolButton(value=ui_symbols.reset) - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) - process_btn= ui_components.ToolButton(value=ui_symbols.preview) + enabled_cb = gr.Checkbox(value= i == 0, label="Enabled") + process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') + model_id = gr.Dropdown(label="Model", choices=lite.list_models(), value='None') + ui_common.create_refresh_button(model_id, lite.list_models, lambda: {"choices": lite.list_models(refresh=True)}, 'refresh_lite_models') + model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) + reset_btn = ui_components.ToolButton(value=ui_symbols.reset) + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) + image_preview = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=False, height=128, width=128, visible=False) + process_btn= ui_components.ToolButton(value=ui_symbols.preview) lite_ui_units.append(unit_ui) units.append(unit.Unit( unit_type = 'lite', @@ -576,6 +575,7 @@ def create_ui(_blocks: gr.Blocks=None): preview_process = preview_process, preview_btn = process_btn, image_upload = image_upload, + image_preview = image_preview, extra_controls = extra_controls, ) ) @@ -593,16 +593,15 @@ def create_ui(_blocks: gr.Blocks=None): gr.Slider(label="Reference adain weight", minimum=0.0, maximum=2.0, step=0.05, value=1.0, interactive=True), ] for i in range(1): # can only have one reference unit - with gr.Accordion(f'Reference unit {i+1}', visible=True) as unit_ui: + with gr.Accordion(f'Reference unit {i+1}', visible=True, elem_classes='control-unit') as unit_ui: with gr.Row(): - with gr.Column(): - with gr.Row(): - enabled_cb = gr.Checkbox(value= i == 0, label="Enabled", visible=False) - model_id = gr.Dropdown(label="Reference", choices=reference.list_models(), value='Reference', visible=False) - model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, visible=False) - reset_btn = ui_components.ToolButton(value=ui_symbols.reset) - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) - process_btn= ui_components.ToolButton(value=ui_symbols.preview) + enabled_cb = gr.Checkbox(value= i == 0, label="Enabled", visible=False) + model_id = gr.Dropdown(label="Reference", choices=reference.list_models(), value='Reference', visible=False) + model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, visible=False) + reset_btn = ui_components.ToolButton(value=ui_symbols.reset) + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) + image_preview = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=False, height=128, width=128, visible=False) + process_btn= ui_components.ToolButton(value=ui_symbols.preview) units.append(unit.Unit( unit_type = 'reference', result_txt = result_txt, @@ -615,6 +614,7 @@ def create_ui(_blocks: gr.Blocks=None): preview_process = preview_process, preview_btn = process_btn, image_upload = image_upload, + image_preview = image_preview, extra_controls = extra_controls, ) )