separate model type detection for base and refiner

CHANGELOG.md

@@ -20,9 +20,10 @@
     thanks @disty0  
   - add model **precompile** option (when model compile is enbled)  
   - **extra network** folder info caching  
-    results in much faster startup when you have large number of extra networks
+    results in much faster startup when you have large number of extra networks  
   - fix extra networks previews  
-  - fix gradio gallery
+  - fix gradio gallery  
+  - css fixes  
 
 ## Update for 2023-08-20
 

modules/processing_diffusers.py

@@ -129,23 +129,27 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
         negative_pooled = None
         prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2)
         if shared.opts.prompt_attention in {'Compel parser', 'Full parser'}:
-            prompt_embed, pooled, negative_embed, negative_pooled = prompt_parser_diffusers.compel_encode_prompts(model, prompts, negative_prompts,
-                                                                                                                  prompts_2, negative_prompts_2,
-                                                                                                                  is_refiner, kwargs.pop("clip_skip", None))
+            prompt_embed, pooled, negative_embed, negative_pooled = prompt_parser_diffusers.compel_encode_prompts(model, prompts, negative_prompts, prompts_2, negative_prompts_2, is_refiner, kwargs.pop("clip_skip", None))
         if 'prompt' in possible:
             if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and prompt_embed is not None:
                 args['prompt_embeds'] = prompt_embed
-                if shared.sd_model_type == "sdxl":
+                if not is_refiner and shared.sd_model_type == "sdxl":
                     args['pooled_prompt_embeds'] = pooled
-                    args['prompt_2'] = None #Cannot pass prompts when passing embeds
+                    # args['prompt_2'] = None # Cannot pass prompts when passing embeds
+                if is_refiner and shared.sd_refiner_type == "sdxl":
+                    args['pooled_prompt_embeds'] = pooled
+                    # args['prompt_2'] = None # Cannot pass prompts when passing embeds
             else:
                 args['prompt'] = prompts
         if 'negative_prompt' in possible:
             if hasattr(model, 'text_encoder') and 'negative_prompt_embeds' in possible and negative_embed is not None:
                 args['negative_prompt_embeds'] = negative_embed
-                if shared.sd_model_type == "sdxl":
+                if not is_refiner and shared.sd_model_type == "sdxl":
                     args['negative_pooled_prompt_embeds'] = negative_pooled
-                    args['negative_prompt_2'] = None
+                    # args['negative_prompt_2'] = None
+                if is_refiner and shared.sd_refiner_type == "sdxl":
+                    args['negative_pooled_prompt_embeds'] = negative_pooled
+                    # args['negative_prompt_2'] = None
             else:
                 args['negative_prompt'] = negative_prompts
         if 'num_inference_steps' in possible:

modules/prompt_parser_diffusers.py

@@ -81,7 +81,11 @@ def compel_encode_prompt(
         shared.log.warning(f"Prompt parser: Compel not supported: {type(pipeline).__name__}")
         return (None, None, None, None)
 
-    if shared.sd_model_type == "sdxl":
+    if not is_refiner and shared.sd_model_type == "sdxl":
+        embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED
+        if clip_skip is not None and clip_skip > 1:
+            shared.log.warning(f"Prompt parser SDXL unsupported: clip_skip={clip_skip}")
+    elif is_refiner and shared.sd_refiner_type == "sdxl":
         embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED
         if clip_skip is not None and clip_skip > 1:
             shared.log.warning(f"Prompt parser SDXL unsupported: clip_skip={clip_skip}")
@@ -105,29 +109,31 @@ def compel_encode_prompt(
         device=shared.device
     )
 
-    if shared.sd_model_type == "sdxl":
-        compel_te2 = Compel(
-            tokenizer=pipeline.tokenizer_2,
-            text_encoder=pipeline.text_encoder_2,
-            returned_embeddings_type=embedding_type,
-            requires_pooled=True,
-            device=shared.device
-        )
-        if not is_refiner:
-            positive_te1 = compel_te1(prompt)
-            positive_te2, positive_pooled = compel_te2(prompt_2)
-            positive = torch.cat((positive_te1, positive_te2), dim=-1)
-            negative_te1 = compel_te1(negative_prompt)
-            negative_te2, negative_pooled = compel_te2(negative_prompt_2)
-            negative = torch.cat((negative_te1, negative_te2), dim=-1)
-        else:
-            positive, positive_pooled = compel_te2(prompt)
-            negative, negative_pooled = compel_te2(negative_prompt)
+    if not is_refiner and shared.sd_model_type == "sdxl":
+        compel_te2 = Compel(tokenizer=pipeline.tokenizer_2, text_encoder=pipeline.text_encoder_2, returned_embeddings_type=embedding_type, requires_pooled=True, device=shared.device)
+        positive_te1 = compel_te1(prompt)
+        positive_te2, positive_pooled = compel_te2(prompt_2)
+        positive = torch.cat((positive_te1, positive_te2), dim=-1)
+        negative_te1 = compel_te1(negative_prompt)
+        negative_te2, negative_pooled = compel_te2(negative_prompt_2)
+        negative = torch.cat((negative_te1, negative_te2), dim=-1)
+
         parsed = compel_te1.parse_prompt_string(prompt)
         debug(f"Prompt parser Compel: {parsed}")
         [prompt_embed, negative_embed] = compel_te2.pad_conditioning_tensors_to_same_length([positive, negative])
         return prompt_embed, positive_pooled, negative_embed, negative_pooled
 
+    if is_refiner and shared.sd_refiner_type == "sdxl":
+        compel_te2 = Compel(tokenizer=pipeline.tokenizer_2, text_encoder=pipeline.text_encoder_2, returned_embeddings_type=embedding_type, requires_pooled=True, device=shared.device)
+        positive, positive_pooled = compel_te2(prompt)
+        negative, negative_pooled = compel_te2(negative_prompt)
+
+        parsed = compel_te1.parse_prompt_string(prompt)
+        debug(f"Prompt parser Compel: {parsed}")
+        [prompt_embed, negative_embed] = compel_te2.pad_conditioning_tensors_to_same_length([positive, negative])
+        return prompt_embed, positive_pooled, negative_embed, negative_pooled
+
+    # neither base+sdxl nor refiner+sdxl
     positive, negative = compel_te1(prompt), compel_te1(negative_prompt)
     [prompt_embed, negative_embed] = compel_te1.pad_conditioning_tensors_to_same_length([positive, negative])
     return prompt_embed, None, negative_embed, None

modules/shared.py

@@ -1020,7 +1020,25 @@ class Shared(sys.modules[__name__].__class__): # this class is here to provide s
             model_type = 'unknown'
         return model_type
 
+    @property
+    def sd_refiner_type(self):
+        try:
+            if backend == Backend.ORIGINAL:
+                model_type = 'ldm'
+            elif "StableDiffusionXL" in self.sd_refiner.__class__.__name__:
+                model_type = 'sdxl'
+            elif "StableDiffusion" in self.sd_refiner.__class__.__name__:
+                model_type = 'sd'
+            elif "Kandinsky" in self.sd_refiner.__class__.__name__:
+                model_type = 'kandinsky'
+            else:
+                model_type = self.sd_refiner.__class__.__name__
+        except Exception:
+            model_type = 'unknown'
+        return model_type
+
 sd_model = None
 sd_refiner = None
 sd_model_type = ''
+sd_refiner_type = ''
 sys.modules[__name__].__class__ = Shared