diff --git a/modules/processing.py b/modules/processing.py index 3ae023fbe..68a1912c3 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -231,6 +231,10 @@ class StableDiffusionProcessing: self.hdr_maximize = hdr_maximize self.hdr_max_center = hdr_max_center self.hdr_max_boundry = hdr_max_boundry + self.prompt_embeds = [] + self.positive_pooleds = [] + self.negative_embeds = [] + self.negative_pooleds = [] @property diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 417139bde..25f6fe646 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -89,8 +89,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if kwargs.get('latents', None) is None: return kwargs kwargs = correction_callback(p, timestep, kwargs) - kwargs["prompt_embeds"] = p.prompt_embeds[step-1] - kwargs["negative_prompt_embeds"] = p.negative_embeds[step-1] + try: + kwargs["prompt_embeds"] = p.prompt_embeds[step + 1].repeat(1, kwargs["prompt_embeds"].shape[0], 1).view( + kwargs["prompt_embeds"].shape[0], kwargs["prompt_embeds"].shape[1], -1) + kwargs["negative_prompt_embeds"] = p.negative_embeds[step + 1].repeat(1, kwargs["negative_prompt_embeds"].shape[0], 1).view( + kwargs["negative_prompt_embeds"].shape[0], kwargs["negative_prompt_embeds"].shape[1], -1) + except: + pass shared.state.current_latent = kwargs['latents'] if shared.cmd_opts.profile and shared.profiler is not None: shared.profiler.step() @@ -295,10 +300,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro possible = signature.parameters.keys() generator_device = devices.cpu if shared.opts.diffusers_generator_device == "cpu" else shared.device generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds] - # prompt_embed = None - # pooled = None - # negative_embed = None - # negative_pooled = None prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2) parser = 'Fixed attention' if shared.opts.prompt_attention != 'Fixed attention' and 'StableDiffusion' in model.__class__.__name__: @@ -312,20 +313,16 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro errors.display(e, 'Prompt parser encode') if 'prompt' in possible: if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and p.prompt_embeds[0] is not None: - # if type(pooled) == list: - # pooled = pooled[0] - # if type(negative_pooled) == list: - # negative_pooled = p.negative_pooleds[0][0] args['prompt_embeds'] = p.prompt_embeds[0] if 'XL' in model.__class__.__name__: - args['pooled_prompt_embeds'] = p.positive_pooleds[0][0] + args['pooled_prompt_embeds'] = p.positive_pooleds[0] else: args['prompt'] = prompts if 'negative_prompt' in possible: if hasattr(model, 'text_encoder') and 'negative_prompt_embeds' in possible and p.negative_embeds[0] is not None: args['negative_prompt_embeds'] = p.negative_embeds[0] if 'XL' in model.__class__.__name__: - args['negative_pooled_prompt_embeds'] = p.negative_pooleds[0][0] + args['negative_pooled_prompt_embeds'] = p.negative_pooleds[0] else: args['negative_prompt'] = negative_prompts if hasattr(model, 'scheduler') and hasattr(model.scheduler, 'noise_sampler_seed') and hasattr(model.scheduler, 'noise_sampler'): @@ -345,7 +342,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro args['callback'] = diffusers_callback_legacy elif 'callback_on_step_end_tensor_inputs' in possible: args['callback_on_step_end'] = diffusers_callback - args['callback_on_step_end_tensor_inputs'] = ['latents'] + args['callback_on_step_end_tensor_inputs'] = ['latents', 'prompt_embeds', 'negative_prompt_embeds'] for arg in kwargs: if arg in possible: # add kwargs args[arg] = kwargs[arg] diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index ceba53977..16fab82d3 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -58,22 +58,22 @@ class DiffusersTextualInversionManager(BaseTextualInversionManager): debug(f'Prompt: expand={prompt}') return self.pipe.tokenizer.encode(prompt, add_special_tokens=False) -def get_prompt_schedule(prompt, steps, step): +def get_prompt_schedule(prompt, steps): temp = [] schedule = prompt_parser.get_learned_conditioning_prompt_schedules([prompt], steps)[0] for chunk in schedule: for s in range(steps): - if s + 1 <= chunk[0]: + if len(temp) < s + 1 <= chunk[0]: temp.append(chunk[1]) - return temp[step-1] + return temp def encode_prompts(pipe, p, prompts: list, negative_prompts: list, steps: int, step: int = 1, clip_skip: typing.Optional[int] = None): if 'StableDiffusion' not in pipe.__class__.__name__ and 'DemoFusion': shared.log.warning(f"Prompt parser not supported: {pipe.__class__.__name__}") return None, None, None, None else: - positive_schedule = get_prompt_schedule(prompts[0]) - negative_schedule = get_prompt_schedule(negative_prompts[0]) + positive_schedule = get_prompt_schedule(prompts[0], steps) + negative_schedule = get_prompt_schedule(negative_prompts[0], steps) p.prompt_embeds = [] p.positive_pooleds = [] @@ -85,7 +85,6 @@ def encode_prompts(pipe, p, prompts: list, negative_prompts: list, steps: int, positive_schedule[i], negative_schedule[i], clip_skip) - if prompt_embed is not None: p.prompt_embeds.append(torch.cat([prompt_embed]*len(prompts), dim=0)) if negative_embed is not None: