diff --git a/CHANGELOG.md b/CHANGELOG.md index a16e00faa..cb04ff4fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,9 +20,10 @@ thanks @disty0 - add model **precompile** option (when model compile is enbled) - **extra network** folder info caching - results in much faster startup when you have large number of extra networks + results in much faster startup when you have large number of extra networks - fix extra networks previews - - fix gradio gallery + - fix gradio gallery + - css fixes ## Update for 2023-08-20 diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 0dc3cdd7d..436d98287 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -129,23 +129,27 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro negative_pooled = None prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2) if shared.opts.prompt_attention in {'Compel parser', 'Full parser'}: - prompt_embed, pooled, negative_embed, negative_pooled = prompt_parser_diffusers.compel_encode_prompts(model, prompts, negative_prompts, - prompts_2, negative_prompts_2, - is_refiner, kwargs.pop("clip_skip", None)) + prompt_embed, pooled, negative_embed, negative_pooled = prompt_parser_diffusers.compel_encode_prompts(model, prompts, negative_prompts, prompts_2, negative_prompts_2, is_refiner, kwargs.pop("clip_skip", None)) if 'prompt' in possible: if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and prompt_embed is not None: args['prompt_embeds'] = prompt_embed - if shared.sd_model_type == "sdxl": + if not is_refiner and shared.sd_model_type == "sdxl": args['pooled_prompt_embeds'] = pooled - args['prompt_2'] = None #Cannot pass prompts when passing embeds + # args['prompt_2'] = None # Cannot pass prompts when passing embeds + if is_refiner and shared.sd_refiner_type == "sdxl": + args['pooled_prompt_embeds'] = pooled + # args['prompt_2'] = None # Cannot pass prompts when passing embeds else: args['prompt'] = prompts if 'negative_prompt' in possible: if hasattr(model, 'text_encoder') and 'negative_prompt_embeds' in possible and negative_embed is not None: args['negative_prompt_embeds'] = negative_embed - if shared.sd_model_type == "sdxl": + if not is_refiner and shared.sd_model_type == "sdxl": args['negative_pooled_prompt_embeds'] = negative_pooled - args['negative_prompt_2'] = None + # args['negative_prompt_2'] = None + if is_refiner and shared.sd_refiner_type == "sdxl": + args['negative_pooled_prompt_embeds'] = negative_pooled + # args['negative_prompt_2'] = None else: args['negative_prompt'] = negative_prompts if 'num_inference_steps' in possible: diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 72da532b8..4bfda29ad 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -81,7 +81,11 @@ def compel_encode_prompt( shared.log.warning(f"Prompt parser: Compel not supported: {type(pipeline).__name__}") return (None, None, None, None) - if shared.sd_model_type == "sdxl": + if not is_refiner and shared.sd_model_type == "sdxl": + embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED + if clip_skip is not None and clip_skip > 1: + shared.log.warning(f"Prompt parser SDXL unsupported: clip_skip={clip_skip}") + elif is_refiner and shared.sd_refiner_type == "sdxl": embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED if clip_skip is not None and clip_skip > 1: shared.log.warning(f"Prompt parser SDXL unsupported: clip_skip={clip_skip}") @@ -105,29 +109,31 @@ def compel_encode_prompt( device=shared.device ) - if shared.sd_model_type == "sdxl": - compel_te2 = Compel( - tokenizer=pipeline.tokenizer_2, - text_encoder=pipeline.text_encoder_2, - returned_embeddings_type=embedding_type, - requires_pooled=True, - device=shared.device - ) - if not is_refiner: - positive_te1 = compel_te1(prompt) - positive_te2, positive_pooled = compel_te2(prompt_2) - positive = torch.cat((positive_te1, positive_te2), dim=-1) - negative_te1 = compel_te1(negative_prompt) - negative_te2, negative_pooled = compel_te2(negative_prompt_2) - negative = torch.cat((negative_te1, negative_te2), dim=-1) - else: - positive, positive_pooled = compel_te2(prompt) - negative, negative_pooled = compel_te2(negative_prompt) + if not is_refiner and shared.sd_model_type == "sdxl": + compel_te2 = Compel(tokenizer=pipeline.tokenizer_2, text_encoder=pipeline.text_encoder_2, returned_embeddings_type=embedding_type, requires_pooled=True, device=shared.device) + positive_te1 = compel_te1(prompt) + positive_te2, positive_pooled = compel_te2(prompt_2) + positive = torch.cat((positive_te1, positive_te2), dim=-1) + negative_te1 = compel_te1(negative_prompt) + negative_te2, negative_pooled = compel_te2(negative_prompt_2) + negative = torch.cat((negative_te1, negative_te2), dim=-1) + parsed = compel_te1.parse_prompt_string(prompt) debug(f"Prompt parser Compel: {parsed}") [prompt_embed, negative_embed] = compel_te2.pad_conditioning_tensors_to_same_length([positive, negative]) return prompt_embed, positive_pooled, negative_embed, negative_pooled + if is_refiner and shared.sd_refiner_type == "sdxl": + compel_te2 = Compel(tokenizer=pipeline.tokenizer_2, text_encoder=pipeline.text_encoder_2, returned_embeddings_type=embedding_type, requires_pooled=True, device=shared.device) + positive, positive_pooled = compel_te2(prompt) + negative, negative_pooled = compel_te2(negative_prompt) + + parsed = compel_te1.parse_prompt_string(prompt) + debug(f"Prompt parser Compel: {parsed}") + [prompt_embed, negative_embed] = compel_te2.pad_conditioning_tensors_to_same_length([positive, negative]) + return prompt_embed, positive_pooled, negative_embed, negative_pooled + + # neither base+sdxl nor refiner+sdxl positive, negative = compel_te1(prompt), compel_te1(negative_prompt) [prompt_embed, negative_embed] = compel_te1.pad_conditioning_tensors_to_same_length([positive, negative]) return prompt_embed, None, negative_embed, None diff --git a/modules/shared.py b/modules/shared.py index 4fce33412..38e82091d 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -1020,7 +1020,25 @@ class Shared(sys.modules[__name__].__class__): # this class is here to provide s model_type = 'unknown' return model_type + @property + def sd_refiner_type(self): + try: + if backend == Backend.ORIGINAL: + model_type = 'ldm' + elif "StableDiffusionXL" in self.sd_refiner.__class__.__name__: + model_type = 'sdxl' + elif "StableDiffusion" in self.sd_refiner.__class__.__name__: + model_type = 'sd' + elif "Kandinsky" in self.sd_refiner.__class__.__name__: + model_type = 'kandinsky' + else: + model_type = self.sd_refiner.__class__.__name__ + except Exception: + model_type = 'unknown' + return model_type + sd_model = None sd_refiner = None sd_model_type = '' +sd_refiner_type = '' sys.modules[__name__].__class__ = Shared