diff --git a/modules/control/adapters.py b/modules/control/adapters.py index 38269f779..263dcef5a 100644 --- a/modules/control/adapters.py +++ b/modules/control/adapters.py @@ -1,6 +1,6 @@ import os import time -from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, T2IAdapter, MultiAdapter, StableDiffusionAdapterPipeline, StableDiffusionXLAdapterPipeline +from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, T2IAdapter, MultiAdapter, StableDiffusionAdapterPipeline, StableDiffusionXLAdapterPipeline # pylint: disable=unused-import from modules.shared import log from modules import errors diff --git a/modules/control/reference.py b/modules/control/reference.py index 4d84aeca5..8116274a7 100644 --- a/modules/control/reference.py +++ b/modules/control/reference.py @@ -5,7 +5,7 @@ from modules.control.proc.reference_sd15 import StableDiffusionReferencePipeline from modules.control.proc.reference_sdxl import StableDiffusionXLReferencePipeline -what = 'ControlNet-XS' +what = 'Reference' def list_models(): diff --git a/modules/control/run.py b/modules/control/run.py index 3dbe7114f..b28963d16 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -193,22 +193,20 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo pass shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset current pipeline - if not has_models and (unit_type == 'reference' or unit_type == 'adapter' or unit_type == 'controlnet' or unit_type == 'xs'): # run in img2img mode - if len(active_strength) > 0: - p.strength = active_strength[0] - pipe = diffusers.AutoPipelineForImage2Image.from_pipe(shared.sd_model) # use set_diffuser_pipe - elif unit_type == 'adapter' and has_models: + debug(f'Control: run type={unit_type} models={has_models}') + if unit_type == 'adapter' and has_models: p.extra_generation_params["Control mode"] = 'Adapter' p.extra_generation_params["Control conditioning"] = use_conditioning - p.adapter_conditioning_scale = use_conditioning + p.task_args['adapter_conditioning_scale'] = use_conditioning instance = adapters.AdapterPipeline(selected_models, shared.sd_model) pipe = instance.pipeline elif unit_type == 'controlnet' and has_models: p.extra_generation_params["Control mode"] = 'ControlNet' p.extra_generation_params["Control conditioning"] = use_conditioning - p.controlnet_conditioning_scale = use_conditioning - p.control_guidance_start = active_start[0] if len(active_start) == 1 else list(active_start) - p.control_guidance_end = active_end[0] if len(active_end) == 1 else list(active_end) + p.task_args['controlnet_conditioning_scale'] = use_conditioning + p.task_args['control_guidance_start'] = active_start[0] if len(active_start) == 1 else list(active_start) + p.task_args['control_guidance_end'] = active_end[0] if len(active_end) == 1 else list(active_end) + p.task_args['guess_mode'] = p.guess_mode instance = controlnets.ControlNetPipeline(selected_models, shared.sd_model) pipe = instance.pipeline elif unit_type == 'xs' and has_models: @@ -222,16 +220,17 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo elif unit_type == 'reference': p.extra_generation_params["Control mode"] = 'Reference' p.extra_generation_params["Control attention"] = p.attention - p.reference_attn = 'Attention' in p.attention - p.reference_adain = 'Adain' in p.attention - p.attention_auto_machine_weight = p.query_weight - p.gn_auto_machine_weight = p.adain_weight - p.style_fidelity = p.fidelity + p.task_args['reference_attn'] = 'Attention' in p.attention + p.task_args['reference_adain'] = 'Adain' in p.attention + p.task_args['attention_auto_machine_weight'] = p.query_weight + p.task_args['gn_auto_machine_weight'] = p.adain_weight + p.task_args['style_fidelity'] = p.fidelity instance = reference.ReferencePipeline(shared.sd_model) pipe = instance.pipeline - else: - shared.log.error(f'Control: unknown unit type: {unit_type}') - pipe = None + else: # run in img2img mode + if len(active_strength) > 0: + p.strength = active_strength[0] + pipe = diffusers.AutoPipelineForImage2Image.from_pipe(shared.sd_model) # use set_diffuser_pipe debug(f'Control pipeline: class={pipe.__class__} args={vars(p)}') t1, t2, t3 = time.time(), 0, 0 status = True @@ -353,20 +352,20 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo # pipeline output = None if pipe is not None: # run new pipeline - if not has_models and (unit_type == 'reference' or unit_type == 'controlnet' or unit_type == 'adapter' or unit_type == 'xs'): # run in img2img mode + if not has_models and (unit_type == 'controlnet' or unit_type == 'adapter' or unit_type == 'xs'): # run in img2img mode if p.image is None: if hasattr(p, 'init_images'): del p.init_images shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset current pipeline else: p.init_images = [processed_image] # pylint: disable=attribute-defined-outside-init - processed_image.save('/tmp/test.png') shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) # reset current pipeline else: if hasattr(p, 'init_images'): del p.init_images shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset current pipeline - debug(f'Control exec pipeline: class={pipe.__class__} args={vars(p)}') + debug(f'Control exec pipeline: class={pipe.__class__} p={vars(p)}') + debug(f'Control exec pipeline: class={pipe.__class__} args={p.task_args}') processed: processing.Processed = processing.process_images(p) # run actual pipeline output = processed.images if processed is not None else None # output = pipe(**vars(p)).images # alternative direct pipe exec call diff --git a/modules/lora b/modules/lora index 0908c5414..0a52b83c6 160000 --- a/modules/lora +++ b/modules/lora @@ -1 +1 @@ -Subproject commit 0908c5414dadfe2185669e2d657d542161647b79 +Subproject commit 0a52b83c6a4c87d6375dc6d4d55ad78c39f9f666 diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 9dfba3b71..a9796e1ea 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -304,7 +304,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro 'width': p.width if hasattr(p, 'width') else None, 'height': p.height if hasattr(p, 'height') else None, } - debug(f'Diffusers task args: {task_args}') + debug(f'Diffusers task specific args: {task_args}') return task_args def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs): @@ -374,9 +374,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if arg in possible: args[arg] = task_kwargs[arg] task_args = getattr(p, 'task_args', {}) + debug(f'Diffusers task args: {task_args}') for k, v in task_args.items(): if k in possible: args[k] = v + else: + debug(f'Diffusers unknown task args: {k}={v}') hypertile_set(p, hr=len(getattr(p, 'init_images', [])) > 0) clean = args.copy() diff --git a/t2i.py b/t2i.py new file mode 100644 index 000000000..0c596169c --- /dev/null +++ b/t2i.py @@ -0,0 +1,77 @@ +import torch +import diffusers +from PIL import Image +from rich import print + +model_id = "runwayml/stable-diffusion-v1-5" +print(f'torch=={torch.__version__} diffusers=={diffusers.__version__}') + +adapters = [ + 'TencentARC/t2iadapter_canny_sd15v2', + # 'TencentARC/t2iadapter_depth_sd15v2', + # 'TencentARC/t2iadapter_zoedepth_sd15v1', + # 'TencentARC/t2iadapter_openpose_sd14v1', + # 'TencentARC/t2iadapter_sketch_sd15v2', +] +seeds = [42] + +print(f'loading: {model_id}') +base = diffusers.StableDiffusionPipeline.from_pretrained(model_id, variant="fp16", cache_dir='/mnt/d/Models/Diffusers').to('cuda') +image = Image.new('RGB', (512,512), 0) # input is irrelevant, so just creating blank image +print('loaded') + +def callback(step: int, timestep: int, latents: torch.FloatTensor): + print(f'callback: step={step} timestep={timestep} latents={latents.shape}') + +for adapter_id in adapters: + print(f'loading: {adapter_id}') + adapter = diffusers.T2IAdapter.from_pretrained('TencentARC/t2iadapter_depth_sd15v2', cache_dir='/mnt/d/Models/Diffusers') + pipe = diffusers.StableDiffusionAdapterPipeline( + vae=base.vae, + text_encoder=base.text_encoder, + tokenizer=base.tokenizer, + unet=base.unet, + scheduler=base.scheduler, + requires_safety_checker=False, + safety_checker=None, + feature_extractor=None, + adapter=adapter, + ).to('cuda') + output = pipe(prompt=['test'], negative_prompt=['test'], num_inference_steps=20, image=image) # ok + print(f'adapter: {adapter_id} {output}') + pipe.scheduler = diffusers.EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.scheduler.config['num_train_timesteps'] = 1000 + pipe.scheduler.config['beta_start'] = 0.00085 + pipe.scheduler.config['beta_end'] = 0.012 + pipe.scheduler.config['beta_schedule'] = 'scaled_linear' + pipe.scheduler.config['prediction_type'] = 'epsilon' + pipe.scheduler.config['rescale_betas_zero_snr'] = False + output = pipe( + prompt=['test'], + negative_prompt=['test'], + num_inference_steps=20, + image=image, + callback=callback, + callback_steps=1, + output_type='latent', + eta=1.0, + clip_skip=1, + guidance_scale=6, + generator=[torch.Generator('cpu').manual_seed(seed) for seed in seeds], + ) + print(f'adapter: {adapter_id} {output}') + +""" +'callback_steps': 1, +'callback': .diffusers_callback_legacy at 0x7f4569259a80>, + +'guidance_scale': 6, +'generator': [], +'num_inference_steps': 20, + +'eta': 1.0, +'clip_skip': 1, +'image': } + +Given groups=1, weight of size [320, 64, 3, 3], expected input[1, 192, 64, 64] to have 64 channels, but got 192 channels instead +""" diff --git a/wiki b/wiki index 5ce9b8d0b..840269c12 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 5ce9b8d0b798194e108094701bc55589fd01cbd1 +Subproject commit 840269c12e0faae36d8b013070360fcd37ee24fc