Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-17 08:19:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

control set task args

Browse Source
This commit is contained in:
Vladimir Mandic
2023-12-21 10:57:11 -05:00
parent 787e9b0b22
commit aeccc28d60
7 changed files with 104 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/control/adapters.py
+1 -1
View File
@@ -1,6 +1,6 @@
import os
import time
from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, T2IAdapter, MultiAdapter, StableDiffusionAdapterPipeline, StableDiffusionXLAdapterPipeline
from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, T2IAdapter, MultiAdapter, StableDiffusionAdapterPipeline, StableDiffusionXLAdapterPipeline # pylint: disable=unused-import
from modules.shared import log
from modules import errors
modules/control/reference.py
+1 -1
View File
@@ -5,7 +5,7 @@ from modules.control.proc.reference_sd15 import StableDiffusionReferencePipeline
from modules.control.proc.reference_sdxl import StableDiffusionXLReferencePipeline
what = 'ControlNet-XS'
what = 'Reference'
def list_models():
modules/control/run.py
+19 -20
View File
@@ -193,22 +193,20 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
pass
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset current pipeline
if not has_models and (unit_type == 'reference' or unit_type == 'adapter' or unit_type == 'controlnet' or unit_type == 'xs'): # run in img2img mode
if len(active_strength) > 0:
p.strength = active_strength[0]
pipe = diffusers.AutoPipelineForImage2Image.from_pipe(shared.sd_model) # use set_diffuser_pipe
elif unit_type == 'adapter' and has_models:
debug(f'Control: run type={unit_type} models={has_models}')
if unit_type == 'adapter' and has_models:
p.extra_generation_params["Control mode"] = 'Adapter'
p.extra_generation_params["Control conditioning"] = use_conditioning
p.adapter_conditioning_scale = use_conditioning
p.task_args['adapter_conditioning_scale'] = use_conditioning
instance = adapters.AdapterPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
elif unit_type == 'controlnet' and has_models:
p.extra_generation_params["Control mode"] = 'ControlNet'
p.extra_generation_params["Control conditioning"] = use_conditioning
p.controlnet_conditioning_scale = use_conditioning
p.control_guidance_start = active_start[0] if len(active_start) == 1 else list(active_start)
p.control_guidance_end = active_end[0] if len(active_end) == 1 else list(active_end)
p.task_args['controlnet_conditioning_scale'] = use_conditioning
p.task_args['control_guidance_start'] = active_start[0] if len(active_start) == 1 else list(active_start)
p.task_args['control_guidance_end'] = active_end[0] if len(active_end) == 1 else list(active_end)
p.task_args['guess_mode'] = p.guess_mode
instance = controlnets.ControlNetPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
elif unit_type == 'xs' and has_models:
@@ -222,16 +220,17 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
elif unit_type == 'reference':
p.extra_generation_params["Control mode"] = 'Reference'
p.extra_generation_params["Control attention"] = p.attention
p.reference_attn = 'Attention' in p.attention
p.reference_adain = 'Adain' in p.attention
p.attention_auto_machine_weight = p.query_weight
p.gn_auto_machine_weight = p.adain_weight
p.style_fidelity = p.fidelity
p.task_args['reference_attn'] = 'Attention' in p.attention
p.task_args['reference_adain'] = 'Adain' in p.attention
p.task_args['attention_auto_machine_weight'] = p.query_weight
p.task_args['gn_auto_machine_weight'] = p.adain_weight
p.task_args['style_fidelity'] = p.fidelity
instance = reference.ReferencePipeline(shared.sd_model)
pipe = instance.pipeline
else:
shared.log.error(f'Control: unknown unit type: {unit_type}')
pipe = None
else: # run in img2img mode
if len(active_strength) > 0:
p.strength = active_strength[0]
pipe = diffusers.AutoPipelineForImage2Image.from_pipe(shared.sd_model) # use set_diffuser_pipe
debug(f'Control pipeline: class={pipe.__class__} args={vars(p)}')
t1, t2, t3 = time.time(), 0, 0
status = True
@@ -353,20 +352,20 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
# pipeline
output = None
if pipe is not None: # run new pipeline
if not has_models and (unit_type == 'reference' or unit_type == 'controlnet' or unit_type == 'adapter' or unit_type == 'xs'): # run in img2img mode
if not has_models and (unit_type == 'controlnet' or unit_type == 'adapter' or unit_type == 'xs'): # run in img2img mode
if p.image is None:
if hasattr(p, 'init_images'):
del p.init_images
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset current pipeline
else:
p.init_images = [processed_image] # pylint: disable=attribute-defined-outside-init
processed_image.save('/tmp/test.png')
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) # reset current pipeline
else:
if hasattr(p, 'init_images'):
del p.init_images
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset current pipeline
debug(f'Control exec pipeline: class={pipe.__class__} args={vars(p)}')
debug(f'Control exec pipeline: class={pipe.__class__} p={vars(p)}')
debug(f'Control exec pipeline: class={pipe.__class__} args={p.task_args}')
processed: processing.Processed = processing.process_images(p) # run actual pipeline
output = processed.images if processed is not None else None
# output = pipe(**vars(p)).images # alternative direct pipe exec call
modules/lora
+1 -1
Submodule modules/lora updated: 0908c5414d...0a52b83c6a
modules/processing_diffusers.py
+4 -1
View File
@@ -304,7 +304,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
'width': p.width if hasattr(p, 'width') else None,
'height': p.height if hasattr(p, 'height') else None,
}
debug(f'Diffusers task args: {task_args}')
debug(f'Diffusers task specific args: {task_args}')
return task_args
def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs):
@@ -374,9 +374,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
if arg in possible:
args[arg] = task_kwargs[arg]
task_args = getattr(p, 'task_args', {})
debug(f'Diffusers task args: {task_args}')
for k, v in task_args.items():
if k in possible:
args[k] = v
else:
debug(f'Diffusers unknown task args: {k}={v}')
hypertile_set(p, hr=len(getattr(p, 'init_images', [])) > 0)
clean = args.copy()
t2i.py
+77
View File
@@ -0,0 +1,77 @@
import torch
import diffusers
from PIL import Image
from rich import print
model_id = "runwayml/stable-diffusion-v1-5"
print(f'torch=={torch.__version__} diffusers=={diffusers.__version__}')
adapters = [
'TencentARC/t2iadapter_canny_sd15v2',
# 'TencentARC/t2iadapter_depth_sd15v2',
# 'TencentARC/t2iadapter_zoedepth_sd15v1',
# 'TencentARC/t2iadapter_openpose_sd14v1',
# 'TencentARC/t2iadapter_sketch_sd15v2',
]
seeds = [42]
print(f'loading: {model_id}')
base = diffusers.StableDiffusionPipeline.from_pretrained(model_id, variant="fp16", cache_dir='/mnt/d/Models/Diffusers').to('cuda')
image = Image.new('RGB', (512,512), 0) # input is irrelevant, so just creating blank image
print('loaded')
def callback(step: int, timestep: int, latents: torch.FloatTensor):
print(f'callback: step={step} timestep={timestep} latents={latents.shape}')
for adapter_id in adapters:
print(f'loading: {adapter_id}')
adapter = diffusers.T2IAdapter.from_pretrained('TencentARC/t2iadapter_depth_sd15v2', cache_dir='/mnt/d/Models/Diffusers')
pipe = diffusers.StableDiffusionAdapterPipeline(
vae=base.vae,
text_encoder=base.text_encoder,
tokenizer=base.tokenizer,
unet=base.unet,
scheduler=base.scheduler,
requires_safety_checker=False,
safety_checker=None,
feature_extractor=None,
adapter=adapter,
).to('cuda')
output = pipe(prompt=['test'], negative_prompt=['test'], num_inference_steps=20, image=image) # ok
print(f'adapter: {adapter_id} {output}')
pipe.scheduler = diffusers.EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
pipe.scheduler.config['num_train_timesteps'] = 1000
pipe.scheduler.config['beta_start'] = 0.00085
pipe.scheduler.config['beta_end'] = 0.012
pipe.scheduler.config['beta_schedule'] = 'scaled_linear'
pipe.scheduler.config['prediction_type'] = 'epsilon'
pipe.scheduler.config['rescale_betas_zero_snr'] = False
output = pipe(
prompt=['test'],
negative_prompt=['test'],
num_inference_steps=20,
image=image,
callback=callback,
callback_steps=1,
output_type='latent',
eta=1.0,
clip_skip=1,
guidance_scale=6,
generator=[torch.Generator('cpu').manual_seed(seed) for seed in seeds],
)
print(f'adapter: {adapter_id} {output}')
"""
'callback_steps': 1,
'callback': <function process_diffusers.<locals>.diffusers_callback_legacy at 0x7f4569259a80>,
'guidance_scale': 6,
'generator': [<torch._C.Generator object at 0x7f4562c72370>],
'num_inference_steps': 20,
'eta': 1.0,
'clip_skip': 1,
'image': <PIL.Image.Image image mode=RGB size=512x512 at 0x7F456A271A50>}
Given groups=1, weight of size [320, 64, 3, 3], expected input[1, 192, 64, 64] to have 64 channels, but got 192 channels instead
"""
wiki
+1 -1
Submodule wiki updated: 5ce9b8d0b7...840269c12e