Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 17:24:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix control ipadapter inpaint

Browse Source
This commit is contained in:
Vladimir Mandic
2024-02-05 08:27:28 -05:00
parent 220498c652
commit ef29f1a238
2 changed files with 11 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/control/run.py
+9 -7
View File
@@ -351,6 +351,8 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
p.extra_generation_params["Mask model"] = masking.opts.model if masking.opts.model is not None else None
if len(active_process) > 0:
masked_image = masking.run_mask(input_image=input_image, input_mask=mask, return_type='Masked', invert=p.inpainting_mask_invert==1) if mask is not None else input_image
else:
masked_image = input_image
for i, process in enumerate(active_process): # list[image]
image_mode = 'L' if unit_type == 't2i adapter' and len(active_model) > i and ('Canny' in active_model[i].model_id or 'Sketch' in active_model[i].model_id) else 'RGB' # t2iadapter canny and sketch work in grayscale only
debug(f'Control: i={i+1} process="{process.processor_id}" input={masked_image} override={process.override}')
@@ -427,8 +429,11 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
yield (None, processed_image, f'Control {msg}')
t2 += time.time() - t2
# prepare pipeline
if not has_models and (unit_type == 'controlnet' or unit_type == 't2i adapter' or unit_type == 'xs' or unit_type == 'lite'): # run in txt2img/img2img/inpaint mode
# determine txt2img, img2img, inpaint pipeline
if unit_type == 'reference': # special case
p.is_control = True
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
elif not has_models: # run in txt2img/img2img/inpaint mode
if mask is not None:
p.task_args['strength'] = p.denoising_strength
p.image_mask = mask
@@ -440,10 +445,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
else:
p.init_hr()
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
elif unit_type == 'reference':
p.is_control = True
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
else: # actual control
elif has_models: # actual control
p.is_control = True
if mask is not None:
p.task_args['strength'] = denoising_strength
@@ -457,7 +459,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
p.task_args['image'] = p.init_images # need to set explicitly for txt2img
if unit_type == 'lite':
instance.apply(selected_models, p.init_images, control_conditioning)
if hasattr(p, 'init_images') and p.init_images is None:
if hasattr(p, 'init_images') and p.init_images is None: # delete as its set via task_args
del p.init_images
# ip adapter apply is run in processing.process_images
scripts/image2video.py
+2
View File
@@ -85,6 +85,8 @@ class Script(scripts.Script):
motion_adapter = diffusers.MotionAdapter.from_pretrained(repo_id)
motion_adapter.to(devices.device, devices.dtype)
shared.sd_model = sd_models.switch_pipe(diffusers.PIAPipeline, shared.sd_model, { 'motion_adapter': motion_adapter })
if not ((shared.opts.diffusers_model_cpu_offload or shared.cmd_opts.medvram) or (shared.opts.diffusers_seq_cpu_offload or shared.cmd_opts.lowvram)):
shared.sd_model.to(devices.device, devices.dtype)
if num_frames > 0:
p.task_args['num_frames'] = num_frames
p.task_args['image'] = p.init_images[0]