Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-18 08:44:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

control input control

Browse Source
This commit is contained in:
Vladimir Mandic
2023-12-22 14:15:08 -05:00
parent 4b74d3ebf9
commit 93a3b532d8
11 changed files with 216 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1 -1
View File
@@ -1,6 +1,6 @@
# Change Log for SD.Next
## Update for 2023-12-21
## Update for 2023-12-22
*Note*: based on `diffusers==0.25.0.dev0`
javascript/sdnext.css
+2
View File
@@ -256,6 +256,8 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
font-size: 0.7em; z-index: 50; font-family: monospace; display: none; }
/* control */
#control_input_type { max-width: 18em }
#control_settings .small-accordion .form { min-width: 350px !important }
.control-button { min-height: 42px; max-height: 42px; line-height: 1em; }
.control-tabs>.tab-nav { margin-bottom: 0; margin-top: 0; }
.processor-settings { padding: 0 !important; max-width: 300px; }
modules/control/adapters.py
+3 -1
View File
@@ -77,7 +77,7 @@ class Adapter():
self.model = None
self.model_id = None
def load(self, model_id: str = None):
def load(self, model_id: str = None) -> str:
try:
t0 = time.time()
model_id = model_id or self.model_id
@@ -97,9 +97,11 @@ class Adapter():
t1 = time.time()
self.model_id = model_id
log.debug(f'Control {what} loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}')
return f'{what} loaded model: {model_id}'
except Exception as e:
log.error(f'Control {what} model load failed: id="{model_id}" error={e}')
errors.display(e, f'Control {what} load')
return f'{what} failed to load model: {model_id}'
class AdapterPipeline():
modules/control/controlnets.py
+3 -1
View File
@@ -90,7 +90,7 @@ class ControlNet():
self.model = None
self.model_id = None
def load(self, model_id: str = None):
def load(self, model_id: str = None) -> str:
try:
t0 = time.time()
model_id = model_id or self.model_id
@@ -115,9 +115,11 @@ class ControlNet():
t1 = time.time()
self.model_id = model_id
log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}')
return f'{what} loaded model: {model_id}'
except Exception as e:
log.error(f'Control {what} model load failed: id="{model_id}" error={e}')
errors.display(e, f'Control {what} load')
return f'{what} failed to load model: {model_id}'
class ControlNetPipeline():
modules/control/controlnetsxs.py
+3 -1
View File
@@ -81,7 +81,7 @@ class ControlNetXS():
self.model = None
self.model_id = None
def load(self, model_id: str = None, time_embedding_mix: float = 0.0):
def load(self, model_id: str = None, time_embedding_mix: float = 0.0) -> str:
try:
t0 = time.time()
model_id = model_id or self.model_id
@@ -107,9 +107,11 @@ class ControlNetXS():
t1 = time.time()
self.model_id = model_id
log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}')
return f'{what} loaded model: {model_id}'
except Exception as e:
log.error(f'Control {what} model load failed: id="{model_id}" error={e}')
errors.display(e, f'Control {what} load')
return f'{what} failed to load model: {model_id}'
class ControlNetXSPipeline():
modules/control/processors.py
+6 -4
View File
@@ -131,13 +131,13 @@ class Processor():
self.processor_id = None
self.override = None
def load(self, processor_id: str = None):
def load(self, processor_id: str = None) -> str:
try:
t0 = time.time()
processor_id = processor_id or self.processor_id
if processor_id is None or processor_id == 'None':
self.reset()
return
return ''
from_config = config.get(processor_id, {}).get('load_config', None)
if from_config is not None:
for k, v in from_config.items():
@@ -158,7 +158,7 @@ class Processor():
pose_ckpt = 'https://huggingface.co/yzd-v/DWPose/resolve/main/dw-ll_ucoco_384.pth'
else:
log.error(f'Control processor load failed: id="{processor_id}" error=unknown model type')
return
return f'Processor failed to load: {processor_id}'
self.model = cls(det_ckpt=det_ckpt, pose_config=pose_config, pose_ckpt=pose_ckpt, device="cpu")
elif 'SegmentAnything' in processor_id:
if 'Base' == config['SegmentAnything']['model']:
@@ -167,7 +167,7 @@ class Processor():
self.model = cls.from_pretrained(model_path = 'segments-arnaud/sam_vit_l', filename='sam_vit_l_0b3195.pth', model_type='vit_l', **self.load_config)
else:
log.error(f'Control processor load failed: id="{processor_id}" error=unknown model type')
return
return f'Processor failed to load: {processor_id}'
elif config[processor_id].get('load_config', None) is not None:
self.model = cls.from_pretrained(**self.load_config)
elif config[processor_id]['checkpoint']:
@@ -177,9 +177,11 @@ class Processor():
t1 = time.time()
self.processor_id = processor_id
log.debug(f'Control processor loaded: id="{processor_id}" class={self.model.__class__.__name__} time={t1-t0:.2f}')
return f'Processor loaded: {processor_id}'
except Exception as e:
log.error(f'Control processor load failed: id="{processor_id}" error={e}')
display(e, 'Control processor load')
return f'Processor load filed: {processor_id}'
def __call__(self, image_input: Image):
if self.override is not None:
modules/control/run.py
+86 -37
View File
@@ -59,7 +59,7 @@ def restore_pipeline():
devices.torch_gc()
def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bool,
def control_run(units: List[unit.Unit], inputs, inits, unit_type: str, is_generator: bool, input_type: int,
prompt, negative, styles, steps, sampler_index,
seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w,
cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, full_quality, restore_faces, tiling,
@@ -69,7 +69,7 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate,
):
global pipe, original_pipeline # pylint: disable=global-statement
debug(f'Control {unit_type}: input={inputs}')
debug(f'Control {unit_type}: input={inputs} init={inits} type={input_type}')
if inputs is None or (type(inputs) is list and len(inputs) == 0):
inputs = [None]
output_images: List[Image.Image] = [] # output images
@@ -110,7 +110,7 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
hdr_maximize = hdr_maximize,
hdr_max_center = hdr_max_center,
hdr_max_boundry = hdr_max_boundry,
resize_mode = resize_mode,
resize_mode = resize_mode if resize_name != 'None' else 0,
resize_name = resize_name,
scale_by = scale_by,
selected_scale_tab = selected_scale_tab,
@@ -136,36 +136,36 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
if unit_type == 'adapter' and u.adapter.model is not None:
active_process.append(u.process)
active_model.append(u.adapter)
active_strength.append(u.strength)
active_strength.append(float(u.strength))
p.adapter_conditioning_factor = u.factor
shared.log.debug(f'Control T2I-Adapter unit: process={u.process.processor_id} model={u.adapter.model_id} strength={u.strength} factor={u.factor}')
elif unit_type == 'controlnet' and u.controlnet.model is not None:
active_process.append(u.process)
active_model.append(u.controlnet)
active_strength.append(u.strength)
active_start.append(u.start)
active_end.append(u.end)
active_strength.append(float(u.strength))
active_start.append(float(u.start))
active_end.append(float(u.end))
p.guess_mode = u.guess
shared.log.debug(f'Control ControlNet unit: process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end}')
elif unit_type == 'xs' and u.controlnet.model is not None:
active_process.append(u.process)
active_model.append(u.controlnet)
active_strength.append(u.strength)
active_start.append(u.start)
active_end.append(u.end)
active_strength.append(float(u.strength))
active_start.append(float(u.start))
active_end.append(float(u.end))
p.guess_mode = u.guess
shared.log.debug(f'Control ControlNet-XS unit: process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end}')
elif unit_type == 'reference':
p.override = u.override
p.attention = u.attention
p.query_weight = u.query_weight
p.adain_weight = u.adain_weight
p.query_weight = float(u.query_weight)
p.adain_weight = float(u.adain_weight)
p.fidelity = u.fidelity
shared.log.debug('Control Reference unit')
else:
active_process.append(u.process)
# active_model.append(model)
active_strength.append(u.strength)
active_strength.append(float(u.strength))
p.ops.append('control')
has_models = False
@@ -184,7 +184,6 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
use_conditioning = active_strength[0] if len(active_strength) == 1 else list(active_strength) # strength or list[strength]
else:
pass
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset current pipeline
debug(f'Control: run type={unit_type} models={has_models}')
if unit_type == 'adapter' and has_models:
@@ -193,6 +192,8 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
p.task_args['adapter_conditioning_scale'] = use_conditioning
instance = adapters.AdapterPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: T2I-Adapter does not support separate init image')
elif unit_type == 'controlnet' and has_models:
p.extra_generation_params["Control mode"] = 'ControlNet'
p.extra_generation_params["Control conditioning"] = use_conditioning
@@ -210,6 +211,8 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
p.control_guidance_end = active_end[0] if len(active_end) == 1 else list(active_end)
instance = controlnetsxs.ControlNetXSPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: ControlNet-XS does not support separate init image')
elif unit_type == 'reference':
p.extra_generation_params["Control mode"] = 'Reference'
p.extra_generation_params["Control attention"] = p.attention
@@ -220,6 +223,8 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
p.task_args['style_fidelity'] = p.fidelity
instance = reference.ReferencePipeline(shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: ControlNet-XS does not support separate init image')
else: # run in img2img mode
if len(active_strength) > 0:
p.strength = active_strength[0]
@@ -244,6 +249,10 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
try:
with devices.inference_context():
if isinstance(inputs, str): # only video, the rest is a list
if input_type == 2: # separate init image
if isinstance(inits, str) and inits != inputs:
shared.log.warning('Control: separate init video not support for video input')
input_type = 1
try:
video = cv2.VideoCapture(inputs)
if not video.isOpened():
@@ -268,6 +277,7 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
if frame is not None:
inputs = [Image.fromarray(frame)] # cv2 to pil
for i, input_image in enumerate(inputs):
debug(f'Control Control image: {i + 1} of {len(inputs)}')
if shared.state.skipped:
shared.state.skipped = False
continue
@@ -278,20 +288,37 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
# get input
if isinstance(input_image, str):
try:
input_image = Image.open(input_image)
input_image = Image.open(inputs[i])
except Exception as e:
shared.log.error(f'Control: image open failed: path={input_image} error={e}')
shared.log.error(f'Control: image open failed: path={inputs[i]} type=control error={e}')
continue
# match init input
if input_type == 1:
debug('Control Init image: same as control')
init_image = input_image
elif inits is None:
debug('Control Init image: none')
init_image = None
elif isinstance(inits[i], str):
debug(f'Control: init image: {inits[i]}')
try:
init_image = Image.open(inits[i])
except Exception as e:
shared.log.error(f'Control: image open failed: path={inits[i]} type=init error={e}')
continue
else:
debug(f'Control Init image: {i % len(inits) + 1} of {len(inits)}')
init_image = inits[i % len(inits)]
index += 1
if video is not None and index % (video_skip_frames + 1) != 0:
continue
# resize
if resize_mode != 0 and input_image is not None:
if p.resize_mode != 0 and input_image is not None:
p.extra_generation_params["Control resize"] = f'{resize_time}: {resize_name}'
if resize_mode != 0 and input_image is not None and resize_time == 'Before':
debug(f'Control resize: image={input_image} width={width} height={height} mode={resize_mode} name={resize_name} sequence={resize_time}')
input_image = images.resize_image(resize_mode, input_image, width, height, resize_name)
if p.resize_mode != 0 and input_image is not None and resize_time == 'Before':
debug(f'Control resize: image={input_image} width={width} height={height} mode={p.resize_mode} name={resize_name} sequence={resize_time}')
input_image = images.resize_image(p.resize_mode, input_image, width, height, resize_name)
# process
if input_image is None:
@@ -335,6 +362,20 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
processed_image = [np.array(i) for i in p.image]
processed_image = util.blend(processed_image) # blend all processed images into one
processed_image = Image.fromarray(processed_image)
if unit_type == 'controlnet' and input_type == 1: # Init image same as control
p.task_args['control_image'] = p.image
p.task_args['strength'] = p.denoising_strength
p.task_args['image'] = input_image
elif unit_type == 'controlnet' and input_type == 2: # Separate init image
p.task_args['control_image'] = p.image
p.task_args['strength'] = p.denoising_strength
if init_image is None:
shared.log.warning('Control: separate init image not provided')
p.task_args['image'] = input_image
else:
p.task_args['image'] = init_image
if is_generator:
image_txt = f'{processed_image.width}x{processed_image.height}' if processed_image is not None else 'None'
msg = f'process | {index} of {frames if video is not None else len(inputs)} | {"Image" if video is None else "Frame"} {image_txt}'
@@ -342,23 +383,31 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
yield (None, processed_image, f'Control {msg}')
t2 += time.time() - t2
# prepare pipeline
if hasattr(p, 'init_images'):
del p.init_images # control never uses init_image as-is
if pipe is not None:
if not has_models and (unit_type == 'controlnet' or unit_type == 'adapter' or unit_type == 'xs'): # run in txt2img or img2img mode
if processed_image is not None:
p.init_images = [processed_image]
pipe = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
else:
pipe = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
elif unit_type == 'reference':
pipe = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
else: # actual control
if 'control_image' in p.task_args:
pipe = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) # only controlnet supports img2img
else:
pipe = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
# pipeline
output = None
if pipe is not None: # run new pipeline
if not has_models and (unit_type == 'controlnet' or unit_type == 'adapter' or unit_type == 'xs'): # run in img2img mode
if p.image is None:
if hasattr(p, 'init_images'):
del p.init_images
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset current pipeline
else:
p.init_images = [processed_image] # pylint: disable=attribute-defined-outside-init
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) # reset current pipeline
else:
if hasattr(p, 'init_images'):
del p.init_images
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset current pipeline
debug(f'Control exec pipeline: class={pipe.__class__} p={vars(p)}')
debug(f'Control exec pipeline: class={pipe.__class__} args={p.task_args}')
debug(f'Control exec pipeline: class={pipe.__class__}')
debug(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)}')
debug(f'Control exec pipeline: p={vars(p)}')
debug(f'Control exec pipeline: args={p.task_args}')
processed: processing.Processed = processing.process_images(p) # run actual pipeline
output = processed.images if processed is not None else None
# output = pipe(**vars(p)).images # alternative direct pipe exec call
@@ -371,9 +420,9 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
output_image = output[0]
if output_image is not None:
# resize
if resize_mode != 0 and resize_time == 'After':
debug(f'Control resize: image={input_image} width={width} height={height} mode={resize_mode} name={resize_name} sequence={resize_time}')
output_image = images.resize_image(resize_mode, output_image, width, height, resize_name)
if p.resize_mode != 0 and resize_time == 'After':
debug(f'Control resize: image={input_image} width={width} height={height} mode={p.resize_mode} name={resize_name} sequence={resize_time}')
output_image = images.resize_image(p.resize_mode, output_image, width, height, resize_name)
elif hasattr(p, 'width') and hasattr(p, 'height'):
output_image = output_image.resize((p.width, p.height), Image.Resampling.LANCZOS)
modules/control/unit.py
+5 -4
View File
@@ -32,6 +32,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
image_upload = None,
control_start = None,
control_end = None,
result_txt = None,
extra_controls: list = [], # noqa B006
):
self.enabled = enabled or False
@@ -118,17 +119,17 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
# bind ui controls to properties if present
if self.type == 'adapter':
if model_id is not None:
model_id.change(fn=self.adapter.load, inputs=[model_id], show_progress=True)
model_id.change(fn=self.adapter.load, inputs=[model_id], outputs=[result_txt], show_progress=True)
if extra_controls is not None and len(extra_controls) > 0:
extra_controls[0].change(fn=adapter_extra, inputs=extra_controls)
elif self.type == 'controlnet':
if model_id is not None:
model_id.change(fn=self.controlnet.load, inputs=[model_id], show_progress=True)
model_id.change(fn=self.controlnet.load, inputs=[model_id], outputs=[result_txt], show_progress=True)
if extra_controls is not None and len(extra_controls) > 0:
extra_controls[0].change(fn=controlnet_extra, inputs=extra_controls)
elif self.type == 'xs':
if model_id is not None:
model_id.change(fn=self.controlnet.load, inputs=[model_id, extra_controls[0]], show_progress=True)
model_id.change(fn=self.controlnet.load, inputs=[model_id, extra_controls[0]], outputs=[result_txt], show_progress=True)
if extra_controls is not None and len(extra_controls) > 0:
extra_controls[0].change(fn=controlnetxs_extra, inputs=extra_controls)
elif self.type == 'reference':
@@ -142,7 +143,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
if model_strength is not None:
model_strength.change(fn=strength_change, inputs=[model_strength])
if process_id is not None:
process_id.change(fn=self.process.load, inputs=[process_id], show_progress=True)
process_id.change(fn=self.process.load, inputs=[process_id], outputs=[result_txt], show_progress=True)
if reset_btn is not None:
reset_btn.click(fn=reset, inputs=[], outputs=[enabled_cb, model_id, process_id, model_strength])
if preview_btn is not None:
modules/ui.py
+5 -2
View File
@@ -208,11 +208,14 @@ def create_advanced_inputs(tab):
return cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, full_quality, restore_faces, tiling, hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_center, hdr_max_boundry
def create_resize_inputs(tab, images, time_selector=False, scale_visible=True):
def create_resize_inputs(tab, images, time_selector=False, scale_visible=True, mode=None):
dummy_component = gr.Number(visible=False, value=0)
with gr.Accordion(open=False, label="Resize", elem_classes=["small-accordion"], elem_id=f"{tab}_resize_group"):
with gr.Row():
resize_mode = gr.Radio(label="Resize mode", elem_id=f"{tab}_resize_mode", choices=shared.resize_modes, type="index", value="None")
if mode is not None:
resize_mode = gr.Radio(label="Resize mode", elem_id=f"{tab}_resize_mode", choices=shared.resize_modes, type="index", value=mode, visible=False)
else:
resize_mode = gr.Radio(label="Resize mode", elem_id=f"{tab}_resize_mode", choices=shared.resize_modes, type="index", value='None')
resize_time = gr.Radio(label="Resize order", elem_id=f"{tab}_resize_order", choices=['Before', 'After'], value="Before", visible=time_selector)
with gr.Row():
resize_name = gr.Dropdown(label="Resize method", elem_id=f"{tab}_resize_name", choices=[x.name for x in shared.sd_upscalers], value=opts.upscaler_for_img2img)
modules/ui_control.py
+101 -43
View File
@@ -14,14 +14,14 @@ gr_height = 512
max_units = 10
units: list[unit.Unit] = [] # main state variable
input_source = None
debug = os.environ.get('SD_CONTROL_DEBUG', None) is not None
if debug:
shared.log.trace('Control debug enabled')
input_init = None
debug = shared.log.trace if os.environ.get('SD_CONTROL_DEBUG', None) is not None else lambda *args, **kwargs: None
debug('Trace: CONTROL')
def initialize():
from modules import devices
shared.log.debug(f'Control initialize: models={shared.opts.control_dir} debug={debug}')
shared.log.debug(f'Control initialize: models={shared.opts.control_dir}')
controlnets.cache_dir = os.path.join(shared.opts.control_dir, 'controlnets')
controlnetsxs.cache_dir = os.path.join(shared.opts.control_dir, 'controlnetsxs')
adapters.cache_dir = os.path.join(shared.opts.control_dir, 'adapters')
@@ -37,8 +37,7 @@ def initialize():
def return_controls(res):
# return preview, image, video, gallery, text
if debug:
shared.log.debug(f'Control received: type={type(res)} {res}')
debug(f'Control received: type={type(res)} {res}')
if isinstance(res, str): # error response
return [None, None, None, None, res]
elif isinstance(res, tuple): # standard response received as tuple via control_run->yield(output_images, process_image, result_txt)
@@ -67,7 +66,7 @@ def generate_click(job_id: str, active_tab: str, *args):
shared.mem_mon.reset()
progress.start_task(job_id)
try:
for results in control_run(units, input_source, active_tab, True, *args):
for results in control_run(units, input_source, input_init, active_tab, True, *args):
progress.record_results(job_id, results)
yield return_controls(results)
except Exception as e:
@@ -106,27 +105,30 @@ def get_video(filepath: str):
return msg
def select_input(selected_input):
global input_source # pylint: disable=global-statement
def select_input(selected_input, selected_init, init_type):
debug(f'Control select input: source={selected_input} init={selected_init}, type={init_type}')
global input_source, input_init # pylint: disable=global-statement
input_type = type(selected_input)
status = 'Control input | Unknown'
res = [gr.Tabs.update(selected='out-gallery'), status]
# control inputs
if hasattr(selected_input, 'size'): # image via upload -> image
input_source = [selected_input]
input_type = 'PIL.Image'
shared.log.debug(f'Control input: type={input_type} input={input_source}')
status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}'
return [gr.Tabs.update(selected='out-gallery'), status]
res = [gr.Tabs.update(selected='out-gallery'), status]
elif isinstance(selected_input, gr.components.image.Image): # not likely
input_source = [selected_input.value]
input_type = 'gr.Image'
shared.log.debug(f'Control input: type={input_type} input={input_source}')
return [gr.Tabs.update(selected='out-gallery'), status]
res = [gr.Tabs.update(selected='out-gallery'), status]
elif isinstance(selected_input, str): # video via upload > tmp filepath to video
input_source = selected_input
input_type = 'gr.Video'
shared.log.debug(f'Control input: type={input_type} input={input_source}')
status = get_video(input_source)
return [gr.Tabs.update(selected='out-video'), status]
res = [gr.Tabs.update(selected='out-video'), status]
elif isinstance(selected_input, list): # batch or folder via upload -> list of tmp filepaths
if hasattr(selected_input[0], 'name'):
input_type = 'tempfiles'
@@ -136,10 +138,46 @@ def select_input(selected_input):
input_source = selected_input
status = f'Control input | Images | Files {len(input_source)}'
shared.log.debug(f'Control input: type={input_type} input={input_source}')
return [gr.Tabs.update(selected='out-gallery'), status]
res = [gr.Tabs.update(selected='out-gallery'), status]
else: # unknown
input_source = None
return [gr.Tabs.update(selected='out-gallery'), status]
# init inputs: optional
if init_type == 0: # Control only
input_init = None
elif init_type == 1: # Init image same as control assigned during runtime
input_init = None
elif init_type == 2: # Separate init image
if hasattr(selected_init, 'size'): # image via upload -> image
input_init = [selected_init]
input_type = 'PIL.Image'
shared.log.debug(f'Control input: type={input_type} input={input_init}')
status = f'Control input | Image | Size {selected_init.width}x{selected_init.height} | Mode {selected_init.mode}'
res = [gr.Tabs.update(selected='out-gallery'), status]
elif isinstance(selected_init, gr.components.image.Image): # not likely
input_init = [selected_init.value]
input_type = 'gr.Image'
shared.log.debug(f'Control input: type={input_type} input={input_init}')
res = [gr.Tabs.update(selected='out-gallery'), status]
elif isinstance(selected_init, str): # video via upload > tmp filepath to video
input_init = selected_init
input_type = 'gr.Video'
shared.log.debug(f'Control input: type={input_type} input={input_init}')
status = get_video(input_init)
res = [gr.Tabs.update(selected='out-video'), status]
elif isinstance(selected_init, list): # batch or folder via upload -> list of tmp filepaths
if hasattr(selected_init[0], 'name'):
input_type = 'tempfiles'
input_init = [f.name for f in selected_init] # tempfile
else:
input_type = 'files'
input_init = selected_init
status = f'Control input | Images | Files {len(input_init)}'
shared.log.debug(f'Control input: type={input_type} input={input_init}')
res = [gr.Tabs.update(selected='out-gallery'), status]
else: # unknown
input_init = None
debug(f'Control select input: source={input_source} init={input_init}')
return res
def video_type_change(video_type):
@@ -162,7 +200,13 @@ def create_ui(_blocks: gr.Blocks=None):
prompt, styles, negative, btn_generate, _btn_interrogate, _btn_deepbooru, btn_paste, btn_extra, prompt_counter, btn_prompt_counter, negative_counter, btn_negative_counter = ui.create_toprow(is_img2img=False, id_part='control')
with FormGroup(elem_id="control_interface", equal_height=False):
with gr.Row(elem_id='control_settings'):
resize_mode, resize_name, width, height, scale_by, selected_scale_tab, resize_time = ui.create_resize_inputs('control', [], time_selector=True, scale_visible=False)
with gr.Accordion(open=False, label="Input", elem_id="control_input", elem_classes=["small-accordion"]):
input_type = gr.Radio(label="Input type", choices=['Control only', 'Init image same as control', 'Separate init image'], value='Control only', type='index', elem_id='control_input_type')
denoising_strength = gr.Slider(minimum=0.01, maximum=0.99, step=0.01, label='Denoising strength', value=0.50, elem_id="control_denoising_strength")
show_preview = gr.Checkbox(label="Show preview", value=True, elem_id="control_show_preview")
resize_mode, resize_name, width, height, scale_by, selected_scale_tab, resize_time = ui.create_resize_inputs('control', [], time_selector=True, scale_visible=False, mode='Fixed')
with gr.Accordion(open=False, label="Sampler", elem_id="control_sampler", elem_classes=["small-accordion"]):
sd_samplers.set_samplers()
@@ -172,9 +216,6 @@ def create_ui(_blocks: gr.Blocks=None):
seed, _reuse_seed, subseed, _reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w = ui.create_seed_inputs('control', reuse_visible=False)
cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, full_quality, restore_faces, tiling, hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_center, hdr_max_boundry = ui.create_advanced_inputs('control')
with gr.Accordion(open=False, label="Denoise", elem_id="control_denoise", elem_classes=["small-accordion"]):
denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Denoising strength', value=0.50, elem_id="control_denoising_strength")
with gr.Accordion(open=False, label="Video", elem_id="control_video", elem_classes=["small-accordion"]):
with gr.Row():
video_skip_frames = gr.Slider(minimum=0, maximum=100, step=1, label='Skip input frames', value=0, elem_id="control_video_skip_frames")
@@ -198,8 +239,8 @@ def create_ui(_blocks: gr.Blocks=None):
result_txt = gr.HTML(elem_classes=['control-result'], elem_id='control-result')
with gr.Row(elem_id='control-inputs'):
with gr.Column(scale=9, elem_id='control-input-column'):
gr.HTML('<span id="control-input-button">Input</p>')
with gr.Column(scale=9, elem_id='control-input-column', visible=True) as _column_input:
gr.HTML('<span id="control-input-button">Control input</p>')
with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-input'):
with gr.Tab('Image', id='in-image') as tab_image:
input_image = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=True, tool="editor", height=gr_height)
@@ -209,7 +250,18 @@ def create_ui(_blocks: gr.Blocks=None):
input_batch = gr.File(label="Input", show_label=False, file_count='multiple', file_types=['image'], type='file', interactive=True, height=gr_height)
with gr.Tab('Folder', id='in-folder') as tab_folder:
input_folder = gr.File(label="Input", show_label=False, file_count='directory', file_types=['image'], type='file', interactive=True, height=gr_height)
with gr.Column(scale=9, elem_id='control-output-column'):
with gr.Column(scale=9, elem_id='control-init-column', visible=False) as column_init:
gr.HTML('<span id="control-init-button">Init input</p>')
with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-init'):
with gr.Tab('Image', id='init-image') as tab_image_init:
init_image = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=True, tool="editor", height=gr_height)
with gr.Tab('Video', id='init-video') as tab_video_init:
init_video = gr.Video(label="Input", show_label=False, interactive=True, height=gr_height)
with gr.Tab('Batch', id='init-batch') as tab_batch_init:
init_batch = gr.File(label="Input", show_label=False, file_count='multiple', file_types=['image'], type='file', interactive=True, height=gr_height)
with gr.Tab('Folder', id='init-folder') as tab_folder_init:
init_folder = gr.File(label="Input", show_label=False, file_count='directory', file_types=['image'], type='file', interactive=True, height=gr_height)
with gr.Column(scale=9, elem_id='control-output-column', visible=True) as _column_output:
gr.HTML('<span id="control-output-button">Output</p>')
with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-output') as output_tabs:
with gr.Tab('Gallery', id='out-gallery'):
@@ -218,20 +270,21 @@ def create_ui(_blocks: gr.Blocks=None):
output_image = gr.Image(label="Input", show_label=False, type="pil", interactive=False, tool="editor", height=gr_height)
with gr.Tab('Video', id='out-video'):
output_video = gr.Video(label="Input", show_label=False, height=gr_height)
with gr.Column(scale=9, elem_id='control-preview-column'):
with gr.Column(scale=9, elem_id='control-preview-column', visible=True) as column_preview:
gr.HTML('<span id="control-preview-button">Preview</p>')
with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-preview'):
with gr.Tab('Preview', id='preview-image') as tab_image:
preview_process = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=False, height=gr_height, visible=True)
input_image.change(fn=select_input, inputs=[input_image], outputs=[output_tabs, result_txt])
input_video.change(fn=select_input, inputs=[input_video], outputs=[output_tabs, result_txt])
input_batch.change(fn=select_input, inputs=[input_batch], outputs=[output_tabs, result_txt])
input_folder.change(fn=select_input, inputs=[input_folder], outputs=[output_tabs, result_txt])
tab_image.select(fn=select_input, inputs=[input_image], outputs=[output_tabs, result_txt])
tab_video.select(fn=select_input, inputs=[input_video], outputs=[output_tabs, result_txt])
tab_batch.select(fn=select_input, inputs=[input_batch], outputs=[output_tabs, result_txt])
tab_folder.select(fn=select_input, inputs=[input_folder], outputs=[output_tabs, result_txt])
for ctrl in [input_image, input_video, input_batch, input_folder, init_image, init_video, init_batch, init_folder, tab_image, tab_video, tab_batch, tab_folder, tab_image_init, tab_video_init, tab_batch_init, tab_folder_init]:
inputs = [input_image, init_image, input_type]
outputs = [output_tabs, result_txt]
if hasattr(ctrl, 'change'):
ctrl.change(fn=select_input, inputs=inputs, outputs=outputs)
if hasattr(ctrl, 'select'):
ctrl.select(fn=select_input, inputs=inputs, outputs=outputs)
show_preview.change(fn=lambda x: gr.update(visible=x), inputs=[show_preview], outputs=[column_preview])
input_type.change(fn=lambda x: gr.update(visible=x == 2), inputs=[input_type], outputs=[column_init])
with gr.Tabs(elem_id='control-tabs') as _tabs_control_type:
@@ -252,15 +305,16 @@ def create_ui(_blocks: gr.Blocks=None):
process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
model_id = gr.Dropdown(label="ControlNet", choices=controlnets.list_models(), value='None')
ui_common.create_refresh_button(model_id, controlnets.list_models, lambda: {"choices": controlnets.list_models(refresh=True)}, 'refresh_control_models')
model_strength = gr.Slider(label="Strength", minimum=0.0, maximum=1.0, step=0.1, value=1.0-i/10)
control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.1, value=0)
control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.1, value=1.0)
model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10)
control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0)
control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0)
reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
process_btn= ui_components.ToolButton(value=ui_symbols.preview)
controlnet_ui_units.append(unit_ui)
units.append(unit.Unit(
unit_type = 'controlnet',
result_txt = result_txt,
image_input = input_image,
enabled_cb = enabled_cb,
reset_btn = reset_btn,
@@ -296,15 +350,16 @@ def create_ui(_blocks: gr.Blocks=None):
process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
model_id = gr.Dropdown(label="ControlNet-XS", choices=controlnetsxs.list_models(), value='None')
ui_common.create_refresh_button(model_id, controlnetsxs.list_models, lambda: {"choices": controlnetsxs.list_models(refresh=True)}, 'refresh_control_models')
model_strength = gr.Slider(label="Strength", minimum=0.0, maximum=1.0, step=0.1, value=1.0-i/10)
control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.1, value=0)
control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.1, value=1.0)
model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10)
control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0)
control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0)
reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
process_btn= ui_components.ToolButton(value=ui_symbols.preview)
controlnetxs_ui_units.append(unit_ui)
units.append(unit.Unit(
unit_type = 'xs',
result_txt = result_txt,
image_input = input_image,
enabled_cb = enabled_cb,
reset_btn = reset_btn,
@@ -327,7 +382,7 @@ def create_ui(_blocks: gr.Blocks=None):
gr.HTML('<a href="https://github.com/TencentARC/T2I-Adapter">T2I-Adapter</a>')
with gr.Row():
extra_controls = [
gr.Slider(label="Control factor", minimum=0.0, maximum=1.0, step=0.1, value=1.0, scale=3),
gr.Slider(label="Control factor", minimum=0.0, maximum=1.0, step=0.05, value=1.0, scale=3),
]
num_adaptor_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1)
adaptor_ui_units = [] # list of hidable accordions
@@ -340,13 +395,14 @@ def create_ui(_blocks: gr.Blocks=None):
process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')
model_id = gr.Dropdown(label="Adapter", choices=adapters.list_models(), value='None')
ui_common.create_refresh_button(model_id, adapters.list_models, lambda: {"choices": adapters.list_models(refresh=True)}, 'refresh_adapter_models')
model_strength = gr.Slider(label="Strength", minimum=0.0, maximum=1.0, step=0.1, value=1.0-i/10)
model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10)
reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
process_btn= ui_components.ToolButton(value=ui_symbols.preview)
adaptor_ui_units.append(unit_ui)
units.append(unit.Unit(
unit_type = 'adapter',
result_txt = result_txt,
image_input = input_image,
enabled_cb = enabled_cb,
reset_btn = reset_btn,
@@ -368,9 +424,9 @@ def create_ui(_blocks: gr.Blocks=None):
with gr.Row():
extra_controls = [
gr.Radio(label="Reference context", choices=['Attention', 'Adain', 'Attention Adain'], value='Attention', interactive=True),
gr.Slider(label="Style fidelity", minimum=0.0, maximum=1.0, step=0.1, value=0.5, interactive=True), # prompt vs control importance
gr.Slider(label="Reference query weight", minimum=0.0, maximum=1.0, step=0.1, value=1.0, interactive=True),
gr.Slider(label="Reference adain weight", minimum=0.0, maximum=2.0, step=0.1, value=1.0, interactive=True),
gr.Slider(label="Style fidelity", minimum=0.0, maximum=1.0, step=0.05, value=0.5, interactive=True), # prompt vs control importance
gr.Slider(label="Reference query weight", minimum=0.0, maximum=1.0, step=0.05, value=1.0, interactive=True),
gr.Slider(label="Reference adain weight", minimum=0.0, maximum=2.0, step=0.05, value=1.0, interactive=True),
]
for i in range(1): # can only have one reference unit
with gr.Accordion(f'Reference unit {i+1}', visible=True) as unit_ui:
@@ -379,12 +435,13 @@ def create_ui(_blocks: gr.Blocks=None):
with gr.Row():
enabled_cb = gr.Checkbox(value= i == 0, label="Enabled", visible=False)
model_id = gr.Dropdown(label="Reference", choices=reference.list_models(), value='Reference', visible=False)
model_strength = gr.Slider(label="Strength", minimum=0.0, maximum=1.0, step=0.1, value=1.0, visible=False)
model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, visible=False)
reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
process_btn= ui_components.ToolButton(value=ui_symbols.preview)
units.append(unit.Unit(
unit_type = 'reference',
result_txt = result_txt,
image_input = input_image,
enabled_cb = enabled_cb,
reset_btn = reset_btn,
@@ -443,6 +500,7 @@ def create_ui(_blocks: gr.Blocks=None):
tabs_state = gr.Text(value='none', visible=False)
input_fields = [
input_type,
prompt, negative, styles,
steps, sampler_index,
seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w,
@@ -476,7 +534,7 @@ def create_ui(_blocks: gr.Blocks=None):
bindings = generation_parameters_copypaste.ParamBinding(paste_button=btn_paste, tabname="control", source_text_component=prompt, source_image_component=output_gallery)
generation_parameters_copypaste.register_paste_params_button(bindings)
if debug:
if os.environ.get('SD_CONTROL_DEBUG', None) is not None: # debug only
from modules.control.test import test_processors, test_controlnets, test_adapters, test_xs
gr.HTML('<br><h1>Debug</h1><br>')
with gr.Row():
wiki
+1 -1
Submodule wiki updated: f75201d5bc...9994450f36