Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-14 18:48:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

control units mixing

Browse Source
This commit is contained in:
Vladimir Mandic
2024-01-30 09:27:20 -05:00
parent 20b5d952c9
commit ca51bfeb3e
4 changed files with 56 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+2 -2
View File
@@ -12,11 +12,11 @@ OPTIONAL:
- mixture tiling [pr](https://github.com/huggingface/diffusers/tree/main/examples/community#stable-diffusion-mixture-tiling)
- instaflow [pr](https://github.com/huggingface/diffusers/pull/6057)[repo](https://github.com/gnobitab/RectifiedFlow)
- control api
- face api
- control params paste
- masking api
- preprocess api
## Update for 2023-01-29
## Update for 2023-01-30
Another big release, highlights being:
- A lot more functionality in the **Control** module:
modules/control/run.py
+38 -21
View File
@@ -198,6 +198,9 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
has_models = False
selected_models: List[Union[controlnet.ControlNetModel, xs.ControlNetXSModel, t2iadapter.AdapterModel]] = None
control_conditioning = None
control_guidance_start = None
control_guidance_end = None
if unit_type == 'adapter' or unit_type == 'controlnet' or unit_type == 'xs' or unit_type == 'lite':
if len(active_model) == 0:
selected_models = None
@@ -205,46 +208,48 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
selected_models = active_model[0].model if active_model[0].model is not None else None
p.extra_generation_params["Control model"] = (active_model[0].model_id or '') if active_model[0].model is not None else None
has_models = selected_models is not None
control_conditioning = active_strength[0]
control_guidance_start = active_start[0]
control_guidance_end = active_end[0]
else:
selected_models = [m.model for m in active_model if m.model is not None]
p.extra_generation_params["Control model"] = ', '.join([(m.model_id or '') for m in active_model if m.model is not None])
has_models = len(selected_models) > 0
use_conditioning = active_strength[0] if len(active_strength) == 1 else list(active_strength) # strength or list[strength]
control_conditioning = active_strength[0] if len(active_strength) == 1 else list(active_strength) # strength or list[strength]
control_guidance_start = active_start[0] if len(active_start) == 1 else list(active_start)
control_guidance_end = active_end[0] if len(active_end) == 1 else list(active_end)
p.extra_generation_params["Control conditioning"] = control_conditioning
else:
pass
debug(f'Control: run type={unit_type} models={has_models}')
if unit_type == 'adapter' and has_models:
p.extra_generation_params["Control mode"] = 'T2I-Adapter'
p.extra_generation_params["Control conditioning"] = use_conditioning
p.task_args['adapter_conditioning_scale'] = use_conditioning
p.task_args['adapter_conditioning_scale'] = control_conditioning
instance = t2iadapter.AdapterPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: T2I-Adapter does not support separate init image')
elif unit_type == 'controlnet' and has_models:
p.extra_generation_params["Control mode"] = 'ControlNet'
p.extra_generation_params["Control conditioning"] = use_conditioning
p.task_args['controlnet_conditioning_scale'] = use_conditioning
p.task_args['control_guidance_start'] = active_start[0] if len(active_start) == 1 else list(active_start)
p.task_args['control_guidance_end'] = active_end[0] if len(active_end) == 1 else list(active_end)
p.task_args['controlnet_conditioning_scale'] = control_conditioning
p.task_args['control_guidance_start'] = control_guidance_start
p.task_args['control_guidance_end'] = control_guidance_end
p.task_args['guess_mode'] = p.guess_mode
instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
elif unit_type == 'xs' and has_models:
p.extra_generation_params["Control mode"] = 'ControlNet-XS'
p.extra_generation_params["Control conditioning"] = use_conditioning
p.controlnet_conditioning_scale = use_conditioning
p.control_guidance_start = active_start[0] if len(active_start) == 1 else list(active_start)
p.control_guidance_end = active_end[0] if len(active_end) == 1 else list(active_end)
p.controlnet_conditioning_scale = control_conditioning
p.control_guidance_start = control_guidance_start
p.control_guidance_end = control_guidance_end
instance = xs.ControlNetXSPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: ControlNet-XS does not support separate init image')
elif unit_type == 'lite' and has_models:
p.extra_generation_params["Control mode"] = 'ControlLLLite'
p.extra_generation_params["Control conditioning"] = use_conditioning
p.controlnet_conditioning_scale = use_conditioning
p.controlnet_conditioning_scale = control_conditioning
instance = lite.ControlLLitePipeline(shared.sd_model)
pipe = instance.pipeline
if inits is not None:
@@ -376,7 +381,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
p.height = input_image.height
debug(f'Control: input image={input_image}')
p.image = []
processed_images = []
masked_image = masking.run_mask(input_image=input_image, input_mask=mask, return_type='Masked') if mask is not None else input_image
for i, process in enumerate(active_process): # list[image]
image_mode = 'L' if unit_type == 'adapter' and len(active_model) > i and ('Canny' in active_model[i].model_id or 'Sketch' in active_model[i].model_id) else 'RGB' # t2iadapter canny and sketch work in grayscale only
@@ -390,23 +395,35 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
scale_by=scale_by_before,
)
if processed_image is not None:
p.image.append(processed_image)
processed_images.append(processed_image)
if shared.opts.control_unload_processor:
processors.config[process.processor_id]['dirty'] = True # to force reload
process.model = None
if len(p.image) > 0:
debug(f'Control processed: {len(processed_images)}')
if len(processed_images) > 0:
p.extra_generation_params["Control process"] = [p.processor_id for p in active_process]
if any(img is None for img in p.image):
if any(img is None for img in processed_images):
msg = 'Control: attempting process but output is none'
shared.log.error(f'{msg}: {p.image}')
shared.log.error(f'{msg}: {processed_images}')
restore_pipeline()
return msg
p.init_images = p.image
processed_image = [np.array(i) for i in p.image]
processed_image = [np.array(i) for i in processed_images]
processed_image = util.blend(processed_image) # blend all processed images into one
processed_image = Image.fromarray(processed_image)
if isinstance(selected_models, list) and len(processed_images) == len(selected_models):
debug(f'Control: inputs match: input={len(processed_images)} models={len(selected_models)}')
p.init_images = processed_images
elif isinstance(selected_models, list) and len(processed_images) != len(selected_models):
msg = f'Control: number of inputs does not match: input={len(processed_images)} models={len(selected_models)}'
shared.log.error(msg)
restore_pipeline()
return msg
elif selected_models is not None:
debug('Control: single model - blending images')
p.init_images = [processed_image]
else:
debug('Control processed: using input direct')
processed_image = input_image
if unit_type == 'reference':
@@ -469,7 +486,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
if hasattr(p, 'init_images') and p.init_images is not None:
p.task_args['image'] = p.init_images # need to set explicitly for txt2img
if unit_type == 'lite':
instance.apply(selected_models, p.image, use_conditioning)
instance.apply(selected_models, p.image, control_conditioning)
if hasattr(p, 'init_images') and p.init_images is None:
del p.init_images
modules/control/unit.py
+1 -1
View File
@@ -37,7 +37,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
result_txt = None,
extra_controls: list = [], # noqa B006
):
self.enabled = enabled or True
self.enabled = enabled or False
self.type = unit_type
self.strength = strength or 1.0
self.start = start or 0
modules/ui_control.py
+15 -5
View File
@@ -398,9 +398,10 @@ def create_ui(_blocks: gr.Blocks=None):
num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1)
controlnet_ui_units = [] # list of hidable accordions
for i in range(max_units):
enabled = True if i==0 else False
with gr.Accordion(f'ControlNet unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui:
with gr.Row():
enabled_cb = gr.Checkbox(value=True, label="")
enabled_cb = gr.Checkbox(enabled, container=False, show_label=False)
process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None')
ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, f'refresh_controlnet_models_{i}')
@@ -414,6 +415,7 @@ def create_ui(_blocks: gr.Blocks=None):
controlnet_ui_units.append(unit_ui)
units.append(unit.Unit(
unit_type = 'controlnet',
enabled = enabled,
result_txt = result_txt,
enabled_cb = enabled_cb,
reset_btn = reset_btn,
@@ -451,9 +453,10 @@ def create_ui(_blocks: gr.Blocks=None):
num_adapter_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1)
adapter_ui_units = [] # list of hidable accordions
for i in range(max_units):
enabled = True if i==0 else False
with gr.Accordion(f'T2I-Adapter unit {i+1}', visible= i < num_adapter_units.value, elem_classes='control-unit') as unit_ui:
with gr.Row():
enabled_cb = gr.Checkbox(value=True, label="Enabled")
enabled_cb = gr.Checkbox(enabled, container=False, show_label=False)
process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
model_id = gr.Dropdown(label="Adapter", choices=t2iadapter.list_models(), value='None')
ui_common.create_refresh_button(model_id, t2iadapter.list_models, lambda: {"choices": t2iadapter.list_models(refresh=True)}, f'refresh_adapter_models_{i}')
@@ -465,6 +468,7 @@ def create_ui(_blocks: gr.Blocks=None):
adapter_ui_units.append(unit_ui)
units.append(unit.Unit(
unit_type = 'adapter',
enabled = enabled,
result_txt = result_txt,
enabled_cb = enabled_cb,
reset_btn = reset_btn,
@@ -491,9 +495,10 @@ def create_ui(_blocks: gr.Blocks=None):
num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1)
controlnetxs_ui_units = [] # list of hidable accordions
for i in range(max_units):
enabled = True if i==0 else False
with gr.Accordion(f'ControlNet-XS unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui:
with gr.Row():
enabled_cb = gr.Checkbox(value=True, label="")
enabled_cb = gr.Checkbox(enabled, container=False, show_label=False)
process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
model_id = gr.Dropdown(label="ControlNet-XS", choices=xs.list_models(), value='None')
ui_common.create_refresh_button(model_id, xs.list_models, lambda: {"choices": xs.list_models(refresh=True)}, f'refresh_xs_models_{i}')
@@ -507,6 +512,7 @@ def create_ui(_blocks: gr.Blocks=None):
controlnetxs_ui_units.append(unit_ui)
units.append(unit.Unit(
unit_type = 'xs',
enabled = enabled,
result_txt = result_txt,
enabled_cb = enabled_cb,
reset_btn = reset_btn,
@@ -534,9 +540,10 @@ def create_ui(_blocks: gr.Blocks=None):
num_lite_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1)
lite_ui_units = [] # list of hidable accordions
for i in range(max_units):
enabled = True if i==0 else False
with gr.Accordion(f'Control-LLLite unit {i+1}', visible= i < num_lite_units.value, elem_classes='control-unit') as unit_ui:
with gr.Row():
enabled_cb = gr.Checkbox(value=True, label="Enabled")
enabled_cb = gr.Checkbox(enabled, container=False, show_label=False)
process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
model_id = gr.Dropdown(label="Model", choices=lite.list_models(), value='None')
ui_common.create_refresh_button(model_id, lite.list_models, lambda: {"choices": lite.list_models(refresh=True)}, f'refresh_lite_models_{i}')
@@ -548,6 +555,7 @@ def create_ui(_blocks: gr.Blocks=None):
lite_ui_units.append(unit_ui)
units.append(unit.Unit(
unit_type = 'lite',
enabled = enabled,
result_txt = result_txt,
enabled_cb = enabled_cb,
reset_btn = reset_btn,
@@ -575,9 +583,10 @@ def create_ui(_blocks: gr.Blocks=None):
gr.Slider(label="Reference adain weight", minimum=0.0, maximum=2.0, step=0.05, value=1.0, interactive=True),
]
for i in range(1): # can only have one reference unit
enabled = True if i==0 else False
with gr.Accordion(f'Reference unit {i+1}', visible=True, elem_classes='control-unit') as unit_ui:
with gr.Row():
enabled_cb = gr.Checkbox(value=True, label="Enabled", visible=False)
enabled_cb = gr.Checkbox(enabled, container=False, show_label=False)
model_id = gr.Dropdown(label="Reference", choices=reference.list_models(), value='Reference', visible=False)
model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, visible=False)
reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
@@ -586,6 +595,7 @@ def create_ui(_blocks: gr.Blocks=None):
process_btn= ui_components.ToolButton(value=ui_symbols.preview)
units.append(unit.Unit(
unit_type = 'reference',
enabled = enabled,
result_txt = result_txt,
enabled_cb = enabled_cb,
reset_btn = reset_btn,