Add support for clip_skip to the diffusers back-end.

modules/prompt_parser_diffusers.py

@@ -11,9 +11,7 @@ def convert_to_compel(prompt: str):
         return None
     all_schedules = prompt_parser.get_learned_conditioning_prompt_schedules(
         prompt, 100
-    )[
-        0
-    ]
+    )[0]
     output_list = prompt_parser.parse_prompt_attention(all_schedules[0][1])
     converted_prompt = []
     for subprompt, weight in output_list:
@@ -26,6 +24,13 @@ def convert_to_compel(prompt: str):
     return converted_prompt
 
 
+CLIP_SKIP_MAPPING = {
+    None: ReturnedEmbeddingsType.LAST_HIDDEN_STATES_NORMALIZED,
+    1: ReturnedEmbeddingsType.LAST_HIDDEN_STATES_NORMALIZED,
+    2: ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NORMALIZED,
+}
+
+
 def compel_encode_prompt(
     pipeline: diffusers.StableDiffusionXLPipeline,
     prompt: str,
@@ -33,6 +38,7 @@ def compel_encode_prompt(
     prompt_2: typing.Optional[str] = None,
     negative_prompt_2: typing.Optional[str] = None,
     is_refiner: bool = None,
+    clip_skip: typing.Optional[int] = None,
 ):
     if shared.sd_model_type not in {"sd", "sdxl"}:
         shared.log.warning(
@@ -41,12 +47,17 @@ def compel_encode_prompt(
         return (None, None, None, None)
 
     if shared.sd_model_type == "sdxl":
-      embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED
-    # if clip_skip > 1: How to pass this from processing.py?
-      # embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NORMALIZED
+        embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED
+        if clip_skip is not None:
+            shared.log.debug("CLIP skip ignored as it is unsupported for SDXL")
     else:
-      embedding_type = ReturnedEmbeddingsType.LAST_HIDDEN_STATES_NORMALIZED
-
+        embedding_type = CLIP_SKIP_MAPPING.get(
+            clip_skip, ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NORMALIZED
+        )
+        if clip_skip not in CLIP_SKIP_MAPPING:
+            shared.log.warning(
+                f"Recieved a CLIP skip of {clip_skip}, but only {set(CLIP_SKIP_MAPPING.keys())} is supported."
+            )
 
     if shared.opts.data["prompt_attention"] != "Compel parser":
         prompt = convert_to_compel(prompt)
@@ -80,10 +91,13 @@ def compel_encode_prompt(
             positive, positive_pooled = compel_te2(prompt)
             negative, negative_pooled = compel_te2(negative_prompt)
 
-        shared.log.debug(f"Parsed Compel string: {compel_te1.parse_prompt_string(prompt)}")
-        [prompt_embed, negative_embed] = compel_te2.pad_conditioning_tensors_to_same_length(
-            [positive, negative]
+        shared.log.debug(
+            f"Parsed Compel string: {compel_te1.parse_prompt_string(prompt)}"
         )
+        [
+            prompt_embed,
+            negative_embed,
+        ] = compel_te2.pad_conditioning_tensors_to_same_length([positive, negative])
         return prompt_embed, positive_pooled, negative_embed, negative_pooled
 
     positive, negative = compel_te1(prompt), compel_te1(negative_prompt)
@@ -92,6 +106,7 @@ def compel_encode_prompt(
     )
     return prompt_embed, None, negative_embed, None
 
+
 # LAST_HIDDEN_STATES_NORMALIZED = 0             # SD1/2 regular
 # PENULTIMATE_HIDDEN_STATES_NORMALIZED = 1      # SD1.5 with "clip skip"
-# PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED = 2  # SDXL    
+# PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED = 2  # SDXL