mirror of
https://github.com/vladmandic/automatic
synced 2026-09-19 09:14:35 +02:00
flux controlnet support
This commit is contained in:
Vladimir Mandic
+13
-11
@@ -234,7 +234,8 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
|
||||
active_start.append(float(u.start))
|
||||
active_end.append(float(u.end))
|
||||
p.guess_mode = u.guess
|
||||
shared.log.debug(f'Control ControlNet unit: i={num_units} process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end}')
|
||||
p.control_mode = u.mode
|
||||
shared.log.debug(f'Control ControlNet unit: i={num_units} process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end} mode={u.mode}')
|
||||
elif unit_type == 'xs' and u.controlnet.model is not None:
|
||||
active_process.append(u.process)
|
||||
active_model.append(u.controlnet)
|
||||
@@ -388,7 +389,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
|
||||
video = cv2.VideoCapture(inputs)
|
||||
if not video.isOpened():
|
||||
if is_generator:
|
||||
yield terminate(f'Control: video open failed: path={inputs}')
|
||||
yield terminate(f'Video open failed: path={inputs}')
|
||||
return [], '', '', 'Error: video open failed'
|
||||
frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
fps = int(video.get(cv2.CAP_PROP_FPS))
|
||||
@@ -401,7 +402,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
|
||||
shared.log.debug(f'Control: input video: path={inputs} frames={frames} fps={fps} size={w}x{h} codec={codec}')
|
||||
except Exception as e:
|
||||
if is_generator:
|
||||
yield terminate(f'Control: video open failed: path={inputs} {e}')
|
||||
yield terminate(f'Video open failed: path={inputs} {e}')
|
||||
return [], '', '', 'Error: video open failed'
|
||||
|
||||
while status:
|
||||
@@ -419,7 +420,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
|
||||
if shared.state.interrupted:
|
||||
shared.state.interrupted = False
|
||||
if is_generator:
|
||||
yield terminate('Control interrupted')
|
||||
yield terminate('Interrupted')
|
||||
return [], '', '', 'Interrupted'
|
||||
# get input
|
||||
if isinstance(input_image, str):
|
||||
@@ -508,7 +509,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
|
||||
pass
|
||||
if any(img is None for img in processed_images):
|
||||
if is_generator:
|
||||
yield terminate('Control: attempting process but output is none')
|
||||
yield terminate('Attempting process but output is none')
|
||||
return [], '', '', 'Error: output is none'
|
||||
if len(processed_images) > 1 and len(active_process) != len(active_model):
|
||||
processed_image = [np.array(i) for i in processed_images]
|
||||
@@ -527,7 +528,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
|
||||
p.init_images = processed_images
|
||||
elif isinstance(selected_models, list) and len(processed_images) != len(selected_models):
|
||||
if is_generator:
|
||||
yield terminate(f'Control: number of inputs does not match: input={len(processed_images)} models={len(selected_models)}')
|
||||
yield terminate(f'Number of inputs does not match: input={len(processed_images)} models={len(selected_models)}')
|
||||
return [], '', '', 'Error: number of inputs does not match'
|
||||
elif selected_models is not None:
|
||||
p.init_images = processed_image
|
||||
@@ -542,14 +543,13 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
|
||||
debug(f'Control: process=None image={p.ref_image}')
|
||||
if p.ref_image is None:
|
||||
if is_generator:
|
||||
yield terminate('Control: attempting reference mode but image is none')
|
||||
yield terminate('Attempting reference mode but image is none')
|
||||
return [], '', '', 'Reference mode without image'
|
||||
elif unit_type == 'controlnet' and has_models:
|
||||
if input_type == 0: # Control only
|
||||
if shared.sd_model_type == 'f1':
|
||||
if is_generator:
|
||||
yield terminate('Control: Flux control invalid input type')
|
||||
return [], '', '', 'Flux control invalid input type'
|
||||
p.task_args['control_image'] = p.init_images # TODO flux controlnet mandates this
|
||||
p.task_args['strength'] = p.denoising_strength
|
||||
elif input_type == 1: # Init image same as control
|
||||
p.task_args['control_image'] = p.init_images # switch image and control_image
|
||||
p.task_args['strength'] = p.denoising_strength
|
||||
@@ -602,6 +602,8 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
|
||||
if unit_type == 'lite':
|
||||
p.init_image = [input_image]
|
||||
instance.apply(selected_models, processed_image, control_conditioning)
|
||||
if p.control_mode is not None:
|
||||
p.task_args['control_mode'] = p.control_mode
|
||||
if hasattr(p, 'init_images') and p.init_images is None: # delete empty
|
||||
del p.init_images
|
||||
|
||||
@@ -609,7 +611,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
|
||||
if has_models:
|
||||
if unit_type in ['controlnet', 't2i adapter', 'lite', 'xs'] and p.task_args.get('image', None) is None and getattr(p, 'init_images', None) is None:
|
||||
if is_generator:
|
||||
yield terminate(f'Control: mode={p.extra_generation_params.get("Control mode", None)} input image is none')
|
||||
yield terminate(f'Mode={p.extra_generation_params.get("Control mode", None)} input image is none')
|
||||
return [], '', '', 'Error: Input image is none'
|
||||
|
||||
# resize mask
|
||||
|
||||
@@ -36,6 +36,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
|
||||
image_preview = None,
|
||||
control_start = None,
|
||||
control_end = None,
|
||||
control_mode = None,
|
||||
result_txt = None,
|
||||
extra_controls: list = [],
|
||||
):
|
||||
@@ -46,6 +47,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
|
||||
self.end = end or 1
|
||||
self.start = min(self.start, self.end)
|
||||
self.end = max(self.start, self.end)
|
||||
self.mode = None
|
||||
# processor always exists, adapter and controlnet are optional
|
||||
self.process: processors.Processor = processors.Processor()
|
||||
self.adapter: t2iadapter.Adapter = None
|
||||
@@ -83,6 +85,12 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
|
||||
self.start = min(start, end)
|
||||
self.end = max(start, end)
|
||||
|
||||
def control_mode_change(mode):
|
||||
self.mode = mode - 1 if mode > 0 else None
|
||||
|
||||
def control_mode_show(model_id):
|
||||
return gr.update(visible='union' in model_id.lower())
|
||||
|
||||
def adapter_extra(c1):
|
||||
self.factor = c1
|
||||
|
||||
@@ -156,6 +164,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
|
||||
self.controlnet.load(model_id)
|
||||
else:
|
||||
model_id.change(fn=self.controlnet.load, inputs=[model_id], outputs=[result_txt], show_progress=True)
|
||||
model_id.change(fn=control_mode_show, inputs=[model_id], outputs=[control_mode], show_progress=False)
|
||||
if extra_controls is not None and len(extra_controls) > 0:
|
||||
extra_controls[0].change(fn=controlnet_extra, inputs=extra_controls)
|
||||
elif self.type == 'xs':
|
||||
@@ -202,3 +211,5 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
|
||||
if control_start is not None and control_end is not None:
|
||||
control_start.change(fn=control_change, inputs=[control_start, control_end])
|
||||
control_end.change(fn=control_change, inputs=[control_start, control_end])
|
||||
if control_mode is not None:
|
||||
control_mode.change(fn=control_mode_change, inputs=[control_mode])
|
||||
|
||||
@@ -103,7 +103,7 @@ class Shared(sys.modules[__name__].__class__):
|
||||
model_type = 'sc'
|
||||
elif "AuraFlow" in self.sd_model.__class__.__name__:
|
||||
model_type = 'auraflow'
|
||||
elif "FluxPipeline" in self.sd_model.__class__.__name__:
|
||||
elif "FluxPipeline" in self.sd_model.__class__.__name__ or "FluxControlNetPipeline" in self.sd_model.__class__.__name__:
|
||||
model_type = 'f1'
|
||||
else:
|
||||
model_type = self.sd_model.__class__.__name__
|
||||
|
||||
@@ -456,6 +456,7 @@ class StableDiffusionProcessingControl(StableDiffusionProcessingImg2Img):
|
||||
self.controlnet_conditioning_scale = None
|
||||
self.control_guidance_start = None
|
||||
self.control_guidance_end = None
|
||||
self.control_mode = None
|
||||
self.reference_attn = None
|
||||
self.reference_adain = None
|
||||
self.attention_auto_machine_weight = None
|
||||
|
||||
+18
-13
@@ -689,20 +689,21 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model', offload=True):
|
||||
if shared.opts.no_half_vae:
|
||||
devices.dtype_vae = torch.float32
|
||||
sd_model.vae.to(devices.dtype_vae)
|
||||
shared.log.debug(f'Setting {op} VAE: no-half')
|
||||
shared.log.debug(f'Setting {op} VAE: no-half=True')
|
||||
if hasattr(sd_model, "enable_vae_slicing"):
|
||||
if shared.opts.diffusers_vae_slicing:
|
||||
shared.log.debug(f'Setting {op}: enable VAE slicing')
|
||||
shared.log.debug(f'Setting {op}: slicing=True')
|
||||
sd_model.enable_vae_slicing()
|
||||
else:
|
||||
sd_model.disable_vae_slicing()
|
||||
if hasattr(sd_model, "enable_vae_tiling"):
|
||||
if shared.opts.diffusers_vae_tiling:
|
||||
shared.log.debug(f'Setting {op}: enable VAE tiling')
|
||||
shared.log.debug(f'Setting {op}: tiling=True')
|
||||
sd_model.enable_vae_tiling()
|
||||
else:
|
||||
sd_model.disable_vae_tiling()
|
||||
if hasattr(sd_model, "vqvae"):
|
||||
shared.log.debug(f'Setting {op} VQVAE: upcast=True')
|
||||
sd_model.vqvae.to(torch.float32) # vqvae is producing nans in fp16
|
||||
|
||||
set_diffusers_attention(sd_model)
|
||||
@@ -710,13 +711,13 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model', offload=True):
|
||||
if shared.opts.diffusers_fuse_projections and hasattr(sd_model, 'fuse_qkv_projections'):
|
||||
try:
|
||||
sd_model.fuse_qkv_projections()
|
||||
shared.log.debug(f'Setting {op}: enable fused projections')
|
||||
shared.log.debug(f'Setting {op}: fused-qkv=True')
|
||||
except Exception as e:
|
||||
shared.log.error(f'Error enabling fused projections: {e}')
|
||||
if shared.opts.diffusers_fuse_projections and hasattr(sd_model, 'transformer') and hasattr(sd_model.transformer, 'fuse_qkv_projections'):
|
||||
try:
|
||||
sd_model.transformer.fuse_qkv_projections()
|
||||
shared.log.debug(f'Setting {op}: enable fused projections')
|
||||
shared.log.debug(f'Setting {op}: fused-qkv=True')
|
||||
except Exception as e:
|
||||
shared.log.error(f'Error enabling fused projections: {e}')
|
||||
if shared.opts.diffusers_eval:
|
||||
@@ -730,7 +731,7 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model', offload=True):
|
||||
sd_model = sd_models_compile.dynamic_quantization(sd_model)
|
||||
|
||||
if shared.opts.opt_channelslast and hasattr(sd_model, 'unet'):
|
||||
shared.log.debug(f'Setting {op}: enable channels last')
|
||||
shared.log.debug(f'Setting {op}: channels-last=True')
|
||||
sd_model.unet.to(memory_format=torch.channels_last)
|
||||
|
||||
if offload:
|
||||
@@ -743,13 +744,12 @@ def set_diffuser_offload(sd_model, op: str = 'model'):
|
||||
if sd_model is None:
|
||||
shared.log.warning(f'{op} is not loaded')
|
||||
return
|
||||
shared.log.debug(f'Setting {op}: offload={shared.opts.diffusers_offload_mode}')
|
||||
if not (hasattr(sd_model, "has_accelerate") and sd_model.has_accelerate):
|
||||
sd_model.has_accelerate = False
|
||||
if hasattr(sd_model, "enable_model_cpu_offload"):
|
||||
if shared.opts.diffusers_offload_mode == "model":
|
||||
try:
|
||||
shared.log.debug(f'Setting {op}: enable model CPU offload')
|
||||
shared.log.debug(f'Setting {op}: offload={shared.opts.diffusers_offload_mode}')
|
||||
if shared.opts.diffusers_move_base or shared.opts.diffusers_move_unet or shared.opts.diffusers_move_refiner:
|
||||
shared.opts.diffusers_move_base = False
|
||||
shared.opts.diffusers_move_unet = False
|
||||
@@ -765,7 +765,7 @@ def set_diffuser_offload(sd_model, op: str = 'model'):
|
||||
if hasattr(sd_model, "enable_sequential_cpu_offload"):
|
||||
if shared.opts.diffusers_offload_mode == "sequential":
|
||||
try:
|
||||
shared.log.debug(f'Setting {op}: enable sequential CPU offload')
|
||||
shared.log.debug(f'Setting {op}: offload={shared.opts.diffusers_offload_mode}')
|
||||
if shared.opts.diffusers_move_base or shared.opts.diffusers_move_unet or shared.opts.diffusers_move_refiner:
|
||||
shared.opts.diffusers_move_base = False
|
||||
shared.opts.diffusers_move_unet = False
|
||||
@@ -785,6 +785,7 @@ def set_diffuser_offload(sd_model, op: str = 'model'):
|
||||
shared.log.error(f'Model offload error: mode={shared.opts.diffusers_offload_mode} {e}')
|
||||
if shared.opts.diffusers_offload_mode == "balanced":
|
||||
try:
|
||||
shared.log.debug(f'Setting {op}: offload={shared.opts.diffusers_offload_mode}')
|
||||
sd_model = apply_balanced_offload(sd_model)
|
||||
except Exception as e:
|
||||
shared.log.error(f'Model offload error: mode={shared.opts.diffusers_offload_mode} {e}')
|
||||
@@ -840,8 +841,6 @@ def apply_balanced_offload(sd_model):
|
||||
shared.log.error(f'Balanced offload: module={module_name} {e}')
|
||||
devices.torch_gc(fast=True)
|
||||
|
||||
if not shared.native:
|
||||
return
|
||||
apply_balanced_offload_to_module(sd_model)
|
||||
if hasattr(sd_model, "prior_pipe"):
|
||||
apply_balanced_offload_to_module(sd_model.prior_pipe)
|
||||
@@ -861,6 +860,7 @@ def normalize_device(device):
|
||||
return torch.device(str(device) + ":0")
|
||||
return torch.device(device)
|
||||
|
||||
|
||||
def move_model(model, device=None, force=False):
|
||||
if model is None or device is None:
|
||||
return
|
||||
@@ -1558,11 +1558,16 @@ def set_diffusers_attention(pipe):
|
||||
for module in modules:
|
||||
if module.__class__.__name__ in ['SD3Transformer2DModel']:
|
||||
module.set_attn_processor(p.JointAttnProcessor2_0())
|
||||
elif module.__class__.__name__ in ['HunyuanDiT2DModel', 'FluxTransformer2DModel']:
|
||||
pass
|
||||
elif module.__class__.__name__ in ['FluxTransformer2DModel']:
|
||||
module.set_attn_processor(p.FluxAttnProcessor2_0())
|
||||
elif module.__class__.__name__ in ['HunyuanDiT2DModel']:
|
||||
module.set_attn_processor(p.HunyuanAttnProcessor2_0())
|
||||
else:
|
||||
module.set_attn_processor(attention)
|
||||
|
||||
if 'ControlNet' in pipe.__class__.__name__: # do not replace attention in ControlNet pipelines
|
||||
return
|
||||
shared.log.debug(f"Setting model: attention={shared.opts.cross_attention_optimization}")
|
||||
if shared.opts.cross_attention_optimization == "Disabled":
|
||||
pass # do nothing
|
||||
elif shared.opts.cross_attention_optimization == "Scaled-Dot-Product": # The default set by Diffusers
|
||||
|
||||
@@ -204,6 +204,7 @@ def create_ui(_blocks: gr.Blocks=None):
|
||||
model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=2.0, step=0.01, value=1.0-i/10)
|
||||
control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0)
|
||||
control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0)
|
||||
control_mode = gr.Dropdown(label="Mode", choices=['', 'Canny', 'Tile', 'Depth', 'Blur', 'Pose', 'Gray', 'LQ'], value=0, type='index', visible=False)
|
||||
reset_btn = ui_components.ToolButton(value=ui_symbols.reset)
|
||||
image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'])
|
||||
image_reuse= ui_components.ToolButton(value=ui_symbols.reuse)
|
||||
@@ -226,6 +227,7 @@ def create_ui(_blocks: gr.Blocks=None):
|
||||
image_preview = image_preview,
|
||||
control_start = control_start,
|
||||
control_end = control_end,
|
||||
control_mode = control_mode,
|
||||
extra_controls = extra_controls,
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user