diff --git a/CHANGELOG.md b/CHANGELOG.md index b49188a28..ba3550394 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,6 @@ - Add FreeU for *backend:diffusers* for *backend:original* use extension: - Add HyperTile: -- Switch Diffusers prompt parser - Include chaiNNer extension as submodule - Convert IPEX slicing to generic: @@ -70,6 +69,11 @@ Upgrades are still possible and supported, but above is recommended for best exp to download go to *Models -> Huggingface*: - `stabilityai/sd-x2-latent-upscaler` *(2.2GB)* - `stabilityai/stable-diffusion-x4-upscaler` *(1.7GB)* + - better **Prompt attention** + should better handle more complex prompts + for sdxl, choose which part of prompt goes to second text encoder - just add `TE2:` separator in the prompt + for hires and refiner, second pass prompt is used if present, otherwise primary prompt is used + thanks @AI-Casanova - better **Hires** support for SD and SDXL - better **TI embeddings** support for SD and SDXL faster loading, wider compatibility and support for embeddings with multiple vectors diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 12989dfbd..dbc223625 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -412,8 +412,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op hires_args = set_pipeline_args( model=shared.sd_model, - prompts=prompts, - negative_prompts=negative_prompts, + prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts, + negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts, prompts_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts, negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts, num_inference_steps=int(p.hr_second_pass_steps // p.denoising_strength + 1), diff --git a/modules/prompt_parser.py b/modules/prompt_parser.py index cf8b2b37a..94e846df7 100644 --- a/modules/prompt_parser.py +++ b/modules/prompt_parser.py @@ -304,7 +304,7 @@ def parse_prompt_attention(text): square_brackets = [] if opts.prompt_attention == 'Fixed attention': res = [[text, 1.0]] - debug(f'Prompt parse-attention: {opts.prompt_attention} {res}') + debug(f'Prompt: parser={opts.prompt_attention} {res}') return res elif opts.prompt_attention == 'Compel parser': conjunction = Compel.parse_prompt_string(text) @@ -313,7 +313,7 @@ def parse_prompt_attention(text): res = [] for frag in conjunction.prompts[0].children: res.append([frag.text, frag.weight]) - debug(f'Prompt parse-attention: {opts.prompt_attention} {res}') + debug(f'Prompt: parser={opts.prompt_attention} {res}') return res elif opts.prompt_attention == 'A1111 parser': re_attention = re_attention_v1 @@ -368,7 +368,7 @@ def parse_prompt_attention(text): res.pop(i + 1) else: i += 1 - debug(f'Prompt parse-attention: {opts.prompt_attention} {res}') + debug(f'Prompt: parser={opts.prompt_attention} {res}') return res if __name__ == "__main__": diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index c9218b66a..927aedf41 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -5,9 +5,11 @@ from compel import ReturnedEmbeddingsType from compel.embeddings_provider import BaseTextualInversionManager, EmbeddingsProvider from modules import shared, prompt_parser + debug_output = os.environ.get('SD_PROMPT_DEBUG', None) debug = shared.log.info if debug_output is not None else lambda *args, **kwargs: None + CLIP_SKIP_MAPPING = { None: ReturnedEmbeddingsType.LAST_HIDDEN_STATES_NORMALIZED, 1: ReturnedEmbeddingsType.LAST_HIDDEN_STATES_NORMALIZED, @@ -46,6 +48,7 @@ class DiffusersTextualInversionManager(BaseTextualInversionManager): prompt = prompt.replace(token, replacement) if hasattr(self.pipe, 'embedding_db'): self.pipe.embedding_db.embeddings_used = list(set(self.pipe.embedding_db.embeddings_used)) + debug(f'Prompt: convert={prompt}') return prompt def expand_textual_inversion_token_ids_if_necessary(self, token_ids: typing.List[int]) -> typing.List[int]: @@ -53,16 +56,11 @@ class DiffusersTextualInversionManager(BaseTextualInversionManager): return token_ids prompt = self.pipe.tokenizer.decode(token_ids) prompt = self.maybe_convert_prompt(prompt, self.pipe.tokenizer) - print(prompt) + debug(f'Prompt: expand={prompt}') return self.pipe.tokenizer.encode(prompt, add_special_tokens=False) -def encode_prompts( - pipeline, - prompts: list, - negative_prompts: list, - clip_skip: typing.Optional[int] = None, -): +def encode_prompts(pipeline, prompts: list, negative_prompts: list, clip_skip: typing.Optional[int] = None): if 'StableDiffusion' not in pipeline.__class__.__name__: shared.log.warning(f"Prompt parser not supported: {pipeline.__class__.__name__}") return None, None, None, None @@ -90,12 +88,12 @@ def encode_prompts( def get_prompts_with_weights(prompt: str): - prompt = DiffusersTextualInversionManager(shared.sd_model, - shared.sd_model.tokenizer or shared.sd_model.tokenizer_2).maybe_convert_prompt( - prompt, shared.sd_model.tokenizer or shared.sd_model.tokenizer_2) + manager = DiffusersTextualInversionManager(shared.sd_model, shared.sd_model.tokenizer or shared.sd_model.tokenizer_2) + prompt = manager.maybe_convert_prompt(prompt, shared.sd_model.tokenizer or shared.sd_model.tokenizer_2) texts_and_weights = prompt_parser.parse_prompt_attention(prompt) texts = [t for t, w in texts_and_weights] text_weights = [w for t, w in texts_and_weights] + debug(f'Prompt: weights={texts_and_weights}') return texts, text_weights @@ -109,24 +107,15 @@ def prepare_embedding_providers(pipe, clip_skip): clip_skip = 2 embedding_type = CLIP_SKIP_MAPPING[clip_skip] if hasattr(pipe, "tokenizer") and hasattr(pipe, "text_encoder"): - embeddings_providers.append( - EmbeddingsProvider(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder, - truncate=False, - returned_embeddings_type=embedding_type)) + embedding = EmbeddingsProvider(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder, truncate=False, returned_embeddings_type=embedding_type) + embeddings_providers.append(embedding) if hasattr(pipe, "tokenizer_2") and hasattr(pipe, "text_encoder_2"): - embeddings_providers.append( - EmbeddingsProvider(tokenizer=pipe.tokenizer_2, text_encoder=pipe.text_encoder_2, - truncate=False, - returned_embeddings_type=embedding_type)) + embedding = EmbeddingsProvider(tokenizer=pipe.tokenizer_2, text_encoder=pipe.text_encoder_2, truncate=False, returned_embeddings_type=embedding_type) + embeddings_providers.append(embedding) return embeddings_providers -def get_weighted_text_embeddings_sdxl( - pipe, - prompt: str = "", - neg_prompt: str = "", - clip_skip: int = None -): +def get_weighted_text_embeddings_sdxl(pipe, prompt: str = "", neg_prompt: str = "", clip_skip: int = None): prompt_2 = prompt.split("TE2:")[-1] neg_prompt_2 = neg_prompt.split("TE2:")[-1] prompt = prompt.split("TE2:")[0] @@ -149,21 +138,13 @@ def get_weighted_text_embeddings_sdxl( prompt_embeds = [] negative_prompt_embeds = [] for i in range(len(embedding_providers)): - prompt_embeds.append( - embedding_providers[i].get_embeddings_for_weighted_prompt_fragments(text_batch=[positives[i]], - fragment_weights_batch=[ - positive_weights[i]], - device=pipe.device)) - negative_prompt_embeds.append( - embedding_providers[i].get_embeddings_for_weighted_prompt_fragments(text_batch=[negatives[i]], - fragment_weights_batch=[ - negative_weights[i]], - device=pipe.device)) + embed = embedding_providers[i].get_embeddings_for_weighted_prompt_fragments(text_batch=[positives[i]], fragment_weights_batch=[positive_weights[i]], device=pipe.device) + prompt_embeds.append(embed) + embed = embedding_providers[i].get_embeddings_for_weighted_prompt_fragments(text_batch=[negatives[i]], fragment_weights_batch=[negative_weights[i]],device=pipe.device) + negative_prompt_embeds.append(embed) + prompt_embeds = torch.cat(prompt_embeds, dim=-1) if len(prompt_embeds) > 1 else prompt_embeds[0] negative_prompt_embeds = torch.cat(negative_prompt_embeds, dim=-1) if len(negative_prompt_embeds) > 1 else negative_prompt_embeds[0] - pooled_prompt_embeds = embedding_providers[-1].get_pooled_embeddings(texts=[prompt_2], device=pipe.device) if prompt_embeds.shape[-1] > 768 else None - negative_pooled_prompt_embeds = embedding_providers[-1].get_pooled_embeddings(texts=[neg_prompt_2], - device=pipe.device) if negative_prompt_embeds.shape[-1] > 768 else None + negative_pooled_prompt_embeds = embedding_providers[-1].get_pooled_embeddings(texts=[neg_prompt_2], device=pipe.device) if negative_prompt_embeds.shape[-1] > 768 else None return prompt_embeds, pooled_prompt_embeds, negative_prompt_embeds, negative_pooled_prompt_embeds -