diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index bbe0d5fb5..3217c3c67 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -64,7 +64,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro negative_embed = None negative_pooled = None if shared.opts.data['prompt_attention'] in {'Compel parser', 'Full parser'}: - prompt_embed, pooled, negative_embed, negative_pooled = prompt_parser_diffusers.compel_encode_prompt(model, prompt, negative_prompt, prompt_2, negative_prompt_2, is_refiner) + prompt_embed, pooled, negative_embed, negative_pooled = prompt_parser_diffusers.compel_encode_prompt(model, prompt, negative_prompt, prompt_2, negative_prompt_2, is_refiner, kwargs.pop("clip_skip", None)) if 'prompt' in possible: if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and prompt_embed is not None: args['prompt_embeds'] = prompt_embed diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index d16ffaee5..9ca929053 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -11,9 +11,7 @@ def convert_to_compel(prompt: str): return None all_schedules = prompt_parser.get_learned_conditioning_prompt_schedules( prompt, 100 - )[ - 0 - ] + )[0] output_list = prompt_parser.parse_prompt_attention(all_schedules[0][1]) converted_prompt = [] for subprompt, weight in output_list: @@ -26,6 +24,13 @@ def convert_to_compel(prompt: str): return converted_prompt +CLIP_SKIP_MAPPING = { + None: ReturnedEmbeddingsType.LAST_HIDDEN_STATES_NORMALIZED, + 1: ReturnedEmbeddingsType.LAST_HIDDEN_STATES_NORMALIZED, + 2: ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NORMALIZED, +} + + def compel_encode_prompt( pipeline: diffusers.StableDiffusionXLPipeline, prompt: str, @@ -33,6 +38,7 @@ def compel_encode_prompt( prompt_2: typing.Optional[str] = None, negative_prompt_2: typing.Optional[str] = None, is_refiner: bool = None, + clip_skip: typing.Optional[int] = None, ): if shared.sd_model_type not in {"sd", "sdxl"}: shared.log.warning( @@ -41,12 +47,17 @@ def compel_encode_prompt( return (None, None, None, None) if shared.sd_model_type == "sdxl": - embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED - # if clip_skip > 1: How to pass this from processing.py? - # embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NORMALIZED + embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED + if clip_skip is not None: + shared.log.debug("CLIP skip ignored as it is unsupported for SDXL") else: - embedding_type = ReturnedEmbeddingsType.LAST_HIDDEN_STATES_NORMALIZED - + embedding_type = CLIP_SKIP_MAPPING.get( + clip_skip, ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NORMALIZED + ) + if clip_skip not in CLIP_SKIP_MAPPING: + shared.log.warning( + f"Recieved a CLIP skip of {clip_skip}, but only {set(CLIP_SKIP_MAPPING.keys())} is supported." + ) if shared.opts.data["prompt_attention"] != "Compel parser": prompt = convert_to_compel(prompt) @@ -80,10 +91,13 @@ def compel_encode_prompt( positive, positive_pooled = compel_te2(prompt) negative, negative_pooled = compel_te2(negative_prompt) - shared.log.debug(f"Parsed Compel string: {compel_te1.parse_prompt_string(prompt)}") - [prompt_embed, negative_embed] = compel_te2.pad_conditioning_tensors_to_same_length( - [positive, negative] + shared.log.debug( + f"Parsed Compel string: {compel_te1.parse_prompt_string(prompt)}" ) + [ + prompt_embed, + negative_embed, + ] = compel_te2.pad_conditioning_tensors_to_same_length([positive, negative]) return prompt_embed, positive_pooled, negative_embed, negative_pooled positive, negative = compel_te1(prompt), compel_te1(negative_prompt) @@ -92,6 +106,7 @@ def compel_encode_prompt( ) return prompt_embed, None, negative_embed, None + # LAST_HIDDEN_STATES_NORMALIZED = 0 # SD1/2 regular # PENULTIMATE_HIDDEN_STATES_NORMALIZED = 1 # SD1.5 with "clip skip" -# PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED = 2 # SDXL +# PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED = 2 # SDXL