diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index aa27932e0..cf84055d2 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -53,7 +53,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro return imgs - def set_pipeline_args(model, prompt: str, negative_prompt: str, prompt_2: typing.Optional[str] =None, negative_prompt_2: typing.Optional[str] = None, is_refiner: bool = False, **kwargs): + def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, is_refiner: bool=False, **kwargs): args = {} pipeline = model signature = inspect.signature(type(pipeline).__call__) @@ -65,7 +65,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro negative_embed = None negative_pooled = None if shared.opts.data['prompt_attention'] in {'Compel parser', 'Full parser'}: - prompt_embed, pooled, negative_embed, negative_pooled = prompt_parser_diffusers.compel_encode_prompt(model, prompt, negative_prompt, prompt_2, negative_prompt_2, is_refiner, kwargs.pop("clip_skip", None)) + prompt_embed, pooled, negative_embed, negative_pooled = prompt_parser_diffusers.compel_encode_prompts(model, + prompts, + negative_prompts, + prompts_2, + negative_prompts_2, + is_refiner, + kwargs.pop("clip_skip", None)) if 'prompt' in possible: if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and prompt_embed is not None: args['prompt_embeds'] = prompt_embed @@ -73,7 +79,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro args['pooled_prompt_embeds'] = pooled args['prompt_2'] = None #Cannot pass prompts when passing embeds else: - args['prompt'] = prompt + args['prompt'] = prompts if 'negative_prompt' in possible: if hasattr(model, 'text_encoder') and 'negative_prompt_embeds' in possible and negative_embed is not None: args['negative_prompt_embeds'] = negative_embed @@ -81,7 +87,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro args['negative_pooled_prompt_embeds'] = negative_pooled args['negative_prompt_2'] = None else: - args['negative_prompt'] = negative_prompt + args['negative_prompt'] = negative_prompts if 'num_inference_steps' in possible: args['num_inference_steps'] = p.steps if 'guidance_scale' in possible: @@ -157,10 +163,10 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro refiner_enabled = shared.sd_refiner is not None and p.enable_hr pipe_args = set_pipeline_args( model=shared.sd_model, - prompt=prompts, - negative_prompt=negative_prompts, - prompt_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts, - negative_prompt_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts, + prompts=prompts, + negative_prompts=negative_prompts, + prompts_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts, + negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts, eta=shared.opts.eta_ddim, guidance_rescale=p.diffusers_guidance_rescale, denoising_start=0 if refiner_enabled and p.refiner_start > 0 and p.refiner_start < 1 else None, @@ -211,8 +217,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro for i in range(len(output.images)): pipe_args = set_pipeline_args( model=shared.sd_refiner, - prompt=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts[i], - negative_prompt=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts[i], + prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts[i], + negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts[i], num_inference_steps=p.hr_second_pass_steps, eta=shared.opts.eta_ddim, strength=p.denoising_strength, diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 1f7a4ed42..591998355 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -1,3 +1,4 @@ +import os import typing import torch import diffusers @@ -5,13 +6,14 @@ from compel import Compel, ReturnedEmbeddingsType import modules.shared as shared import modules.prompt_parser as prompt_parser +debug_output = os.environ.get('SD_PROMPT_DEBUG', None) +debug = shared.log.info if debug_output is not None else lambda *args, **kwargs: None + def convert_to_compel(prompt: str): if prompt is None: return None - all_schedules = prompt_parser.get_learned_conditioning_prompt_schedules( - prompt, 100 - )[0] + all_schedules = prompt_parser.get_learned_conditioning_prompt_schedules([prompt], 100)[0] output_list = prompt_parser.parse_prompt_attention(all_schedules[0][1]) converted_prompt = [] for subprompt, weight in output_list: @@ -31,8 +33,35 @@ CLIP_SKIP_MAPPING = { } +def compel_encode_prompts( + pipeline: diffusers.StableDiffusionXLPipeline | diffusers.StableDiffusionPipeline, + prompts: list, + negative_prompts: list, + prompts_2: typing.Optional[list] = None, + negative_prompts_2: typing.Optional[list] = None, + is_refiner: bool = None, + clip_skip: typing.Optional[int] = None, +): + prompt_embeds = [] + positive_pooleds = [] + negative_embeds = [] + negative_pooleds = [] + for i in range(len(prompts)): + prompt_embed, positive_pooled, negative_embed, negative_pooled = compel_encode_prompt(pipeline, prompts[i], negative_prompts[i], prompts_2[i], negative_prompts_2[i], is_refiner, clip_skip) + prompt_embeds.append(prompt_embed) + positive_pooleds.append(positive_pooled) + negative_embeds.append(negative_embed) + negative_pooleds.append(negative_pooled) + + prompt_embeds = torch.cat(prompt_embeds, dim=0) + positive_pooleds = torch.cat(positive_pooleds, dim=0) + negative_embeds = torch.cat(negative_embeds, dim=0) + negative_pooleds = torch.cat(negative_pooleds, dim=0) + return prompt_embeds, positive_pooleds, negative_embeds, negative_pooleds + + def compel_encode_prompt( - pipeline: diffusers.StableDiffusionXLPipeline, + pipeline: diffusers.StableDiffusionXLPipeline | diffusers.StableDiffusionPipeline, prompt: str, negative_prompt: str, prompt_2: typing.Optional[str] = None, @@ -41,24 +70,17 @@ def compel_encode_prompt( clip_skip: typing.Optional[int] = None, ): if shared.sd_model_type not in {"sd", "sdxl"}: - shared.log.warning( - f"Compel encoding not yet supported for {type(pipeline).__name__}." - ) + shared.log.warning(f"Prompt parser: Compel not supported: {type(pipeline).__name__}") return (None, None, None, None) if shared.sd_model_type == "sdxl": embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED - if clip_skip is not None: - shared.log.debug("CLIP skip ignored as it is unsupported for SDXL") + if clip_skip is not None and clip_skip > 1: + shared.log.warning(f"Prompt parser SDXL unsupported: clip_skip={clip_skip}") else: - embedding_type = CLIP_SKIP_MAPPING.get( - clip_skip, ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NORMALIZED - ) + embedding_type = CLIP_SKIP_MAPPING.get(clip_skip, ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NORMALIZED) if clip_skip not in CLIP_SKIP_MAPPING: - shared.log.warning( - f"Recieved a CLIP skip of {clip_skip}, but only {set(CLIP_SKIP_MAPPING.keys())} is supported. " - "Falling back to 2." - ) + shared.log.warning(f"Prompt parser unsupported: clip_skip={clip_skip} expected={set(CLIP_SKIP_MAPPING.keys())}") if shared.opts.data["prompt_attention"] != "Compel parser": prompt = convert_to_compel(prompt) @@ -84,25 +106,17 @@ def compel_encode_prompt( positive_te1 = compel_te1(prompt) positive_te2, positive_pooled = compel_te2(prompt_2) positive = torch.cat((positive_te1, positive_te2), dim=-1) - negative_te1 = compel_te1(negative_prompt) negative_te2, negative_pooled = compel_te2(negative_prompt_2) negative = torch.cat((negative_te1, negative_te2), dim=-1) else: positive, positive_pooled = compel_te2(prompt) negative, negative_pooled = compel_te2(negative_prompt) - - shared.log.debug( - f"Parsed Compel string: {compel_te1.parse_prompt_string(prompt)}" - ) - [ - prompt_embed, - negative_embed, - ] = compel_te2.pad_conditioning_tensors_to_same_length([positive, negative]) + parsed = compel_te1.parse_prompt_string(prompt) + debug(f"Prompt parser Compel: {parsed}") + [prompt_embed, negative_embed] = compel_te2.pad_conditioning_tensors_to_same_length([positive, negative]) return prompt_embed, positive_pooled, negative_embed, negative_pooled positive, negative = compel_te1(prompt), compel_te1(negative_prompt) - [prompt_embed, negative_embed] = compel_te1.pad_conditioning_tensors_to_same_length( - [positive, negative] - ) + [prompt_embed, negative_embed] = compel_te1.pad_conditioning_tensors_to_same_length([positive, negative]) return prompt_embed, None, negative_embed, None