Merge pull request #4964 from vladmandic/fix/prompt-encoding-fixes

Fix/prompt encoding fixes

modules/apg/pipeline_stable_diffusion_apg.py

@@ -404,7 +404,8 @@ class StableDiffusionPipelineAPG(
                 # representations. The `last_hidden_states` that we typically use for
                 # obtaining the final prompt representations passes through the LayerNorm
                 # layer.
-                prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds)
+                # transformers >=5.6 flattened CLIPTextModel; CLIPTextModelWithProjection still nests it under .text_model
+                prompt_embeds = getattr(self.text_encoder, 'text_model', self.text_encoder).final_layer_norm(prompt_embeds)
 
         if self.text_encoder is not None:
             prompt_embeds_dtype = self.text_encoder.dtype

modules/control/units/xs_pipe.py

@@ -1321,7 +1321,8 @@ class StableDiffusionControlNetXSPipeline(
                 # representations. The `last_hidden_states` that we typically use for
                 # obtaining the final prompt representations passes through the LayerNorm
                 # layer.
-                prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds)
+                # transformers >=5.6 flattened CLIPTextModel; CLIPTextModelWithProjection still nests it under .text_model
+                prompt_embeds = getattr(self.text_encoder, 'text_model', self.text_encoder).final_layer_norm(prompt_embeds)
 
         if self.text_encoder is not None:
             prompt_embeds_dtype = self.text_encoder.dtype

modules/options_handler.py

@@ -109,7 +109,8 @@ class Options:
             setattr(self, key, value)
         except RuntimeError:
             return False
-        func = self.data_labels[key].onchange
+        # compatibility_opts (e.g. clip_skip) live in data without a data_labels entry
+        func = self.data_labels[key].onchange if key in self.data_labels else None
         if func is not None:
             try:
                 func()

modules/pag/pipe_sd.py

@@ -619,7 +619,8 @@ class StableDiffusionPAGPipeline(
                 # representations. The `last_hidden_states` that we typically use for
                 # obtaining the final prompt representations passes through the LayerNorm
                 # layer.
-                prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds)
+                # transformers >=5.6 flattened CLIPTextModel; CLIPTextModelWithProjection still nests it under .text_model
+                prompt_embeds = getattr(self.text_encoder, 'text_model', self.text_encoder).final_layer_norm(prompt_embeds)
 
         if self.text_encoder is not None:
             prompt_embeds_dtype = self.text_encoder.dtype

modules/processing.py

@@ -153,11 +153,12 @@ def process_images(p: StableDiffusionProcessing) -> Processed | None:
     for k, v in p.override_settings.copy().items():
         if shared.opts.data.get(k, None) is None and shared.opts.data_labels.get(k, None) is None:
             continue
-        orig = shared.opts.data.get(k, None) or shared.opts.data_labels[k].default
+        # getattr resolves the value via data then data_labels; compat opts (clip_skip) have no data_labels entry
+        orig = getattr(shared.opts, k, None)
         if orig == v or (type(orig) == str and os.path.splitext(orig)[0] == v):
             p.override_settings.pop(k, None)
     for k in p.override_settings.keys():
-        stored_opts[k] = shared.opts.data.get(k, None) or shared.opts.data_labels[k].default
+        stored_opts[k] = getattr(shared.opts, k, None)
     results = None
     try:
         # if no checkpoint override or the override checkpoint can't be found, remove override entry and load opts checkpoint

modules/prompt_parser_diffusers.py

@@ -327,7 +327,9 @@ def compel_hijack(self, token_ids: torch.Tensor, attention_mask: torch.Tensor |
     else:
         hidden_state = text_encoder_output.hidden_states[-clip_skip]
     if normalized:
-        hidden_state = self.text_encoder.text_model.final_layer_norm(hidden_state)
+        # transformers >=5.6 flattened CLIPTextModel; CLIPTextModelWithProjection still nests it under .text_model
+        text_model = getattr(self.text_encoder, 'text_model', self.text_encoder)
+        hidden_state = text_model.final_layer_norm(hidden_state)
     return hidden_state
 
 

modules/prompt_parser_xhinker.py

@@ -216,9 +216,11 @@ def get_weighted_text_embeddings_sd15(
             , generator = torch.Generator(text2img_pipe.device).manual_seed(2)
         ).images[0]
     """
-    original_clip_layers = pipe.text_encoder.text_model.encoder.layers
+    # transformers >=5.6 flattened CLIPTextModel; CLIPTextModelWithProjection still nests it under .text_model
+    clip_text_model = getattr(pipe.text_encoder, 'text_model', pipe.text_encoder)
+    original_clip_layers = clip_text_model.encoder.layers
     if clip_skip > 0:
-        pipe.text_encoder.text_model.encoder.layers = original_clip_layers[:-clip_skip]
+        clip_text_model.encoder.layers = original_clip_layers[:-clip_skip]
 
     eos = pipe.tokenizer.eos_token_id
     prompt_tokens, prompt_weights = get_prompts_tokens_with_weights(
@@ -310,7 +312,7 @@ def get_weighted_text_embeddings_sd15(
 
     # recover clip layers
     if clip_skip > 0:
-        pipe.text_encoder.text_model.encoder.layers = original_clip_layers
+        clip_text_model.encoder.layers = original_clip_layers
 
     return prompt_embeds, neg_prompt_embeds
 

modules/textual_inversion.py

@@ -2,6 +2,7 @@ import os
 import time
 import torch
 import safetensors.torch
+from transformers import AddedToken
 from modules.errorlimiter import limit_errors
 from modules import shared, devices, errors
 from modules.logger import log
@@ -118,13 +119,17 @@ def deref_tokenizers(tokens, tokenizers):
 def insert_tokens(embeddings: list, tokenizers: list):
     """
     Add all tokens to each tokenizer in the list, with one call to each.
+    normalized=False keeps tokens case-sensitive so tokenizer.tokenize surfaces them verbatim;
+    transformers >=5 defaults add_tokens to normalized=True, which lowercases CLIP embedding names
+    and breaks multi-vector expansion (maybe_convert_prompt) in prompt_parser_diffusers.
     """
     tokens = []
     for embedding in embeddings:
         if embedding is not None:
             tokens += embedding.tokens
+    added = [AddedToken(token, normalized=False) for token in tokens]
     for tokenizer in tokenizers:
-        tokenizer.add_tokens(tokens)
+        tokenizer.add_tokens(added)
 
 
 def insert_vectors(embedding, tokenizers, text_encoders, hiddensizes):

scripts/differential_diffusion.py

@@ -1546,7 +1546,8 @@ class StableDiffusionDiffImg2ImgPipeline(DiffusionPipeline):
                 # representations. The `last_hidden_states` that we typically use for
                 # obtaining the final prompt representations passes through the LayerNorm
                 # layer.
-                prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds)
+                # transformers >=5.6 flattened CLIPTextModel; CLIPTextModelWithProjection still nests it under .text_model
+                prompt_embeds = getattr(self.text_encoder, 'text_model', self.text_encoder).final_layer_norm(prompt_embeds)
 
         if self.text_encoder is not None:
             prompt_embeds_dtype = self.text_encoder.dtype