From 90e12b78803e8336be148e43393bf6f0729e51d0 Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Sat, 26 Oct 2024 12:17:58 -0500 Subject: [PATCH 01/40] Initial Prompt Refactor --- modules/processing_args.py | 37 ++++----- modules/processing_callbacks.py | 16 ++-- modules/processing_class.py | 11 +-- modules/prompt_parser_diffusers.py | 126 +++++++++++++++++++++++++++++ 4 files changed, 159 insertions(+), 31 deletions(-) diff --git a/modules/processing_args.py b/modules/processing_args.py index 1ea91fb08..8dd47bd87 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -117,7 +117,8 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 'Flux' in model.__class__.__name__ ): try: - prompt_parser_diffusers.encode_prompts(model, p, prompts, negative_prompts, steps=steps, clip_skip=clip_skip) + # prompt_parser_diffusers.encode_prompts(model, p, prompts, negative_prompts, steps=steps, clip_skip=clip_skip) + p.embedder = prompt_parser_diffusers.PromptEmbedder(prompts, negative_prompts, clip_skip, p) parser = shared.opts.prompt_attention except Exception as e: shared.log.error(f'Prompt parser encode: {e}') @@ -128,27 +129,27 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 if 'prompt' in possible: if 'OmniGen' in model.__class__.__name__: prompts = [p.replace('|image|', '<|image_1|>') for p in prompts] - if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and len(p.prompt_embeds) > 0 and p.prompt_embeds[0] is not None: - args['prompt_embeds'] = p.prompt_embeds[0] + if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and p.embedder is not None: + args['prompt_embeds'] = p.embedder('prompt_embeds') if 'StableCascade' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0: - args['prompt_embeds_pooled'] = p.positive_pooleds[0].unsqueeze(0) - elif 'XL' in model.__class__.__name__ and len(getattr(p, 'positive_pooleds', [])) > 0: - args['pooled_prompt_embeds'] = p.positive_pooleds[0] - elif 'StableDiffusion3' in model.__class__.__name__ and len(getattr(p, 'positive_pooleds', [])) > 0: - args['pooled_prompt_embeds'] = p.positive_pooleds[0] - elif 'Flux' in model.__class__.__name__ and len(getattr(p, 'positive_pooleds', [])) > 0: - args['pooled_prompt_embeds'] = p.positive_pooleds[0] + args['prompt_embeds_pooled'] = p.embedder('positive_pooleds').unsqueeze(0) + elif 'XL' in model.__class__.__name__ and p.embedder is not None: + args['pooled_prompt_embeds'] = p.embedder('positive_pooleds') + elif 'StableDiffusion3' in model.__class__.__name__ and p.embedder is not None: + args['pooled_prompt_embeds'] = p.embedder('positive_pooleds') + elif 'Flux' in model.__class__.__name__ and p.embedder is not None: + args['pooled_prompt_embeds'] = p.embedder('positive_pooleds') else: args['prompt'] = prompts if 'negative_prompt' in possible: - if hasattr(model, 'text_encoder') and 'negative_prompt_embeds' in possible and len(p.negative_embeds) > 0 and p.negative_embeds[0] is not None: - args['negative_prompt_embeds'] = p.negative_embeds[0] - if 'StableCascade' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0: - args['negative_prompt_embeds_pooled'] = p.negative_pooleds[0].unsqueeze(0) - if 'XL' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0: - args['negative_pooled_prompt_embeds'] = p.negative_pooleds[0] - if 'StableDiffusion3' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0: - args['negative_pooled_prompt_embeds'] = p.negative_pooleds[0] + if hasattr(model, 'text_encoder') and 'negative_prompt_embeds' in possible and p.embedder is not None: + args['negative_prompt_embeds'] = p.embedder('negative_embeds') + if 'StableCascade' in model.__class__.__name__ and p.embedder is not None: + args['negative_prompt_embeds_pooled'] = p.embedder('negative_pooleds').unsqueeze(0) + if 'XL' in model.__class__.__name__ and p.embedder is not None: + args['negative_pooled_prompt_embeds'] = p.embedder('negative_pooleds') + if 'StableDiffusion3' in model.__class__.__name__ and p.embedder is not None: + args['negative_pooled_prompt_embeds'] = p.embedder('negative_pooleds') else: if 'PixArtSigmaPipeline' in model.__class__.__name__: # pixart-sigma pipeline throws list-of-list for negative prompt args['negative_prompt'] = negative_prompts[0] diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 47c8e8827..2584ee796 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -67,14 +67,14 @@ def diffusers_callback(pipe, step: int, timestep: int, kwargs: dict): pipe.set_ip_adapter_scale(ip_adapter_scales) if step != getattr(pipe, 'num_timesteps', 0): kwargs = processing_correction.correction_callback(p, timestep, kwargs) - if p.scheduled_prompt and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs: - try: - i = (step + 1) % len(p.prompt_embeds) - kwargs["prompt_embeds"] = p.prompt_embeds[i][0:1].expand(kwargs["prompt_embeds"].shape) - j = (step + 1) % len(p.negative_embeds) - kwargs["negative_prompt_embeds"] = p.negative_embeds[j][0:1].expand(kwargs["negative_prompt_embeds"].shape) - except Exception as e: - shared.log.debug(f"Callback: {e}") + # if p.scheduled_prompt and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs: + # try: + # i = (step + 1) % len(p.prompt_embeds) + # kwargs["prompt_embeds"] = p.prompt_embeds[i][0:1].expand(kwargs["prompt_embeds"].shape) + # j = (step + 1) % len(p.negative_embeds) + # kwargs["negative_prompt_embeds"] = p.negative_embeds[j][0:1].expand(kwargs["negative_prompt_embeds"].shape) + # except Exception as e: + # shared.log.debug(f"Callback: {e}") if step == int(getattr(pipe, 'num_timesteps', 100) * p.cfg_end) and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs: if "PAG" in shared.sd_model.__class__.__name__: pipe._guidance_scale = 1.001 if pipe._guidance_scale > 1 else pipe._guidance_scale # pylint: disable=protected-access diff --git a/modules/processing_class.py b/modules/processing_class.py index 9265ea3cf..d38aae790 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -204,11 +204,12 @@ class StableDiffusionProcessing: self.hdr_color_picker=hdr_color_picker self.hdr_tint_ratio=hdr_tint_ratio # globals - self.scheduled_prompt: bool = False - self.prompt_embeds = [] - self.positive_pooleds = [] - self.negative_embeds = [] - self.negative_pooleds = [] + self.embedder = None + # self.scheduled_prompt: bool = False + # self.prompt_embeds = [] + # self.positive_pooleds = [] + # self.negative_embeds = [] + # self.negative_pooleds = [] @property def sd_model(self): diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index cc814f379..42a6bcbbd 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -7,6 +7,7 @@ from compel.embeddings_provider import BaseTextualInversionManager, EmbeddingsPr from transformers import PreTrainedTokenizer from modules import shared, prompt_parser, devices, sd_models from modules.prompt_parser_xhinker import get_weighted_text_embeddings_sd15, get_weighted_text_embeddings_sdxl_2p, get_weighted_text_embeddings_sd3, get_weighted_text_embeddings_flux1 +from modules.processing_helpers import fix_prompts debug_enabled = os.environ.get('SD_PROMPT_DEBUG', None) debug = shared.log.trace if os.environ.get('SD_PROMPT_DEBUG', None) is not None else lambda *args, **kwargs: None @@ -17,6 +18,131 @@ token_type = None # used by helper get_tokens cache = {} +def prompt_compatible(): + if ( + 'StableDiffusion' not in shared.sd_model.__class__.__name__ and + 'DemoFusion' not in shared.sd_model.__class__.__name__ and + 'StableCascade' not in shared.sd_model.__class__.__name__ and + 'Flux' not in shared.sd_model.__class__.__name__ + ): + shared.log.warning(f"Prompt parser not supported: {shared.sd_model.__class__.__name__}") + return False + return True + + +def prepare_model(): + pipe = shared.sd_model + if shared.opts.diffusers_offload_mode == "balanced": + pipe = sd_models.apply_balanced_offload(pipe) + elif hasattr(pipe, "maybe_free_model_hooks"): + pipe.maybe_free_model_hooks() + devices.torch_gc() + return pipe + + +class PromptEmbedder: + def __init__(self, prompts, negative_prompts, clip_skip, p): + t0 = time.time() + # self.prompts, self.negative_prompts, _, _ = fix_prompts(prompts, negative_prompts, None, None) + self.prompts = prompts + self.negative_prompts = negative_prompts + self.batchsize = len(self.prompts) + self.allsame = self.compare_prompts() # collapses batched prompts to single prompt if same + self.steps = p.steps + self.clip_skip = clip_skip + self.prompt_embeds = [[]] * self.batchsize + self.positive_pooleds = [[]] * self.batchsize + self.negative_embeds = [[]] * self.batchsize + self.negative_pooleds = [[]] * self.batchsize + self.positive_schedule = None + self.negative_schedule = None + self.scheduled_prompt = False + pipe = prepare_model() + # per prompt in batch + for batchidx, (prompt, negative_prompt) in enumerate(zip(self.prompts, self.negative_prompts)): + self.prepare_schedule(prompt, negative_prompt) + if self.scheduled_prompt: + self.scheduled_encode(pipe, batchidx) + else: + self.encode(pipe, prompt, negative_prompt, batchidx) + if self.allsame: + self.duplicate_embeds() + debug(f"Prompt encode: time={(time.time() - t0):.3f}") + + def compare_prompts(self): + same = (self.prompts == [self.prompts[0]] * len(self.prompts) and + self.negative_prompts == [self.negative_prompts[0]] * len(self.negative_prompts)) + if same: + self.prompts = [self.prompts[0]] + self.negative_prompts = [self.negative_prompts[0]] + return same + + def prepare_schedule(self, prompt, negative_prompt): + self.positive_schedule, scheduled = get_prompt_schedule(prompt, self.steps) + self.negative_schedule, neg_scheduled = get_prompt_schedule(negative_prompt, self.steps) + self.scheduled_prompt = scheduled or neg_scheduled + + def scheduled_encode(self, pipe, batchidx): + prompt_dict = {} + for i in range(max(len(self.positive_schedule), len(self.negative_schedule))): + positive_prompt = self.positive_schedule[i % len(self.positive_schedule)] + negative_prompt = self.negative_schedule[i % len(self.negative_schedule)] + # skip repeated scheduled subprompts + idx = prompt_dict.get(positive_prompt+negative_prompt) + if idx is not None: + self.extend_embeds(batchidx, idx) + continue + self.encode(pipe, positive_prompt, negative_prompt, batchidx) + prompt_dict[positive_prompt+negative_prompt] = i + + def extend_embeds(self, batchidx, idx): + self.prompt_embeds[batchidx].append(self.prompt_embeds[batchidx][idx]) + self.negative_embeds[batchidx].append(self.negative_embeds[batchidx][idx]) + if len(self.positive_pooleds[batchidx]) > 0: + self.positive_pooleds[batchidx].append(self.positive_pooleds[batchidx][idx]) + if len(self.negative_pooleds[batchidx]) > 0: + self.negative_pooleds[batchidx].append(self.negative_pooleds[batchidx][idx]) + + def duplicate_embeds(self): + self.prompt_embeds = self.prompt_embeds[0] * self.batchsize + self.positive_pooleds = self.positive_pooleds[0] * self.batchsize + self.negative_embeds = self.negative_embeds[0] * self.batchsize + self.negative_pooleds = self.negative_pooleds[0] * self.batchsize + + def encode(self, pipe, positive_prompt, negative_prompt, batchidx): + if shared.opts.prompt_attention == "xhinker parser" or 'Flux' in pipe.__class__.__name__: + prompt_embed, positive_pooled, negative_embed, negative_pooled = get_xhinker_text_embeddings( + pipe, positive_prompt, negative_prompt, self.clip_skip) + else: + prompt_embed, positive_pooled, negative_embed, negative_pooled = get_weighted_text_embeddings( + pipe, positive_prompt, negative_prompt, self.clip_skip) + if prompt_embed is not None: + self.prompt_embeds[batchidx].append(prompt_embed) + if negative_embed is not None: + self.negative_embeds[batchidx].append(negative_embed) + if positive_pooled is not None: + self.positive_pooleds[batchidx].append(positive_pooled) + if negative_pooled is not None: + self.negative_pooleds[batchidx].append(negative_pooled) + + if debug_enabled: + get_tokens('positive', positive_prompt) + get_tokens('negative', negative_prompt) + pipe = prepare_model() + + def __call__(self, key, step=0): + batch = getattr(self, key) + res = [] + for embed in batch: + if len(embed) == 0: + return None + if len(embed) == 1: + res.append(embed[0]) + else: + res.append(embed[step]) + return torch.stack(res) + + def compel_hijack(self, token_ids: torch.Tensor, attention_mask: typing.Optional[torch.Tensor] = None) -> torch.Tensor: if not devices.same_device(self.text_encoder.device, devices.device): sd_models.move_model(self.text_encoder, devices.device) From f3442abc929c966581a48e74cb4e782bfda2a39d Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Sat, 26 Oct 2024 18:47:11 -0500 Subject: [PATCH 02/40] Prompt LRU Cache --- modules/extra_networks.py | 2 +- modules/prompt_parser_diffusers.py | 44 +++++++++++++++++++++++++++--- modules/shared.py | 3 +- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/modules/extra_networks.py b/modules/extra_networks.py index a574e8469..673549b6b 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -102,8 +102,8 @@ def activate(p, extra_network_data, step=0): except Exception as e: errors.display(e, f"Activating network: type={extra_network_name}") + p.extra_network_data = extra_network_data if stepwise: - p.extra_network_data = extra_network_data shared.opts.data['lora_functional'] = functional diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 42a6bcbbd..ecf0cebd5 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -3,11 +3,11 @@ import math import time import typing import torch +from collections import OrderedDict from compel.embeddings_provider import BaseTextualInversionManager, EmbeddingsProvider from transformers import PreTrainedTokenizer from modules import shared, prompt_parser, devices, sd_models from modules.prompt_parser_xhinker import get_weighted_text_embeddings_sd15, get_weighted_text_embeddings_sdxl_2p, get_weighted_text_embeddings_sd3, get_weighted_text_embeddings_flux1 -from modules.processing_helpers import fix_prompts debug_enabled = os.environ.get('SD_PROMPT_DEBUG', None) debug = shared.log.trace if os.environ.get('SD_PROMPT_DEBUG', None) is not None else lambda *args, **kwargs: None @@ -15,7 +15,7 @@ debug('Trace: PROMPT') orig_encode_token_ids_to_embeddings = EmbeddingsProvider._encode_token_ids_to_embeddings # pylint: disable=protected-access token_dict = None # used by helper get_tokens token_type = None # used by helper get_tokens -cache = {} +cache = OrderedDict() def prompt_compatible(): @@ -57,6 +57,9 @@ class PromptEmbedder: self.positive_schedule = None self.negative_schedule = None self.scheduled_prompt = False + earlyout = self.checkcache(p) + if earlyout: + return pipe = prepare_model() # per prompt in batch for batchidx, (prompt, negative_prompt) in enumerate(zip(self.prompts, self.negative_prompts)): @@ -66,8 +69,41 @@ class PromptEmbedder: else: self.encode(pipe, prompt, negative_prompt, batchidx) if self.allsame: - self.duplicate_embeds() + self.fix_batch_embeds() debug(f"Prompt encode: time={(time.time() - t0):.3f}") + self.checkcache(p) + + def checkcache(self, p): + if shared.opts.sd_textencoder_cache_size == 0: + return False + def flatten(xss): + return [x for xs in xss for x in xs] + + # unpack EN data in case of TE LoRA + en_data = p.extra_network_data + en_data = [idx.items for item in en_data.values() for idx in item] + key = str([self.prompts, self.negative_prompts, self.batchsize, self.clip_skip, self.steps, en_data]) + item = cache.get(key) + if not item: + if not any([flatten(emb) for emb in [self.prompt_embeds, + self.negative_embeds, + self.positive_pooleds, + self.negative_pooleds]]): + return False + else: + cache[key] = {'prompt_embeds': self.prompt_embeds, + 'negative_embeds': self.negative_embeds, + 'positive_pooleds': self.positive_pooleds, + 'negative_pooleds': self.negative_pooleds, + } + debug(f"Prompt cache: Adding {key}") + while len(cache) > int(shared.opts.sd_textencoder_cache_size): + cache.popitem(last=False) + if item: + self.__dict__.update(cache[key]) + cache.move_to_end(key) + debug(f"Prompt cache: Retrieving {key}") + return True def compare_prompts(self): same = (self.prompts == [self.prompts[0]] * len(self.prompts) and @@ -103,7 +139,7 @@ class PromptEmbedder: if len(self.negative_pooleds[batchidx]) > 0: self.negative_pooleds[batchidx].append(self.negative_pooleds[batchidx][idx]) - def duplicate_embeds(self): + def fix_batch_embeds(self): self.prompt_embeds = self.prompt_embeds[0] * self.batchsize self.positive_pooleds = self.positive_pooleds[0] * self.batchsize self.negative_embeds = self.negative_embeds[0] * self.batchsize diff --git a/modules/shared.py b/modules/shared.py index f7be44390..4622f1d2c 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -435,7 +435,8 @@ options_templates.update(options_section(('sd', "Execution & Models"), { "sd_model_dict": OptionInfo('None', "Use separate base dict", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints), "sd_checkpoint_autoload": OptionInfo(True, "Model autoload on start"), "sd_checkpoint_autodownload": OptionInfo(True, "Model auto-download on demand"), - "sd_textencoder_cache": OptionInfo(True, "Cache text encoder results"), + "sd_textencoder_cache": OptionInfo(True, "Cache text encoder results", gr.Checkbox, {"visible": False}), + "sd_textencoder_cache_size": OptionInfo(4, "Text encoder results LRU cache size", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}), "stream_load": OptionInfo(False, "Load models using stream loading method", gr.Checkbox, {"visible": not native }), "model_reuse_dict": OptionInfo(False, "Reuse loaded model dictionary", gr.Checkbox, {"visible": False}), "prompt_mean_norm": OptionInfo(False, "Prompt attention normalization", gr.Checkbox), From 39dfa9cbdbe147f8cc0698cab955b55467ca1918 Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Mon, 28 Oct 2024 22:32:26 -0500 Subject: [PATCH 03/40] Scheduling and cleanup --- modules/processing_args.py | 3 +- modules/processing_callbacks.py | 16 +-- modules/prompt_parser_diffusers.py | 178 +++++------------------------ 3 files changed, 39 insertions(+), 158 deletions(-) diff --git a/modules/processing_args.py b/modules/processing_args.py index 8dd47bd87..34dd97a1c 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -117,7 +117,6 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 'Flux' in model.__class__.__name__ ): try: - # prompt_parser_diffusers.encode_prompts(model, p, prompts, negative_prompts, steps=steps, clip_skip=clip_skip) p.embedder = prompt_parser_diffusers.PromptEmbedder(prompts, negative_prompts, clip_skip, p) parser = shared.opts.prompt_attention except Exception as e: @@ -143,7 +142,7 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 args['prompt'] = prompts if 'negative_prompt' in possible: if hasattr(model, 'text_encoder') and 'negative_prompt_embeds' in possible and p.embedder is not None: - args['negative_prompt_embeds'] = p.embedder('negative_embeds') + args['negative_prompt_embeds'] = p.embedder('negative_prompt_embeds') if 'StableCascade' in model.__class__.__name__ and p.embedder is not None: args['negative_prompt_embeds_pooled'] = p.embedder('negative_pooleds').unsqueeze(0) if 'XL' in model.__class__.__name__ and p.embedder is not None: diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 2584ee796..5c24aead0 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -67,14 +67,14 @@ def diffusers_callback(pipe, step: int, timestep: int, kwargs: dict): pipe.set_ip_adapter_scale(ip_adapter_scales) if step != getattr(pipe, 'num_timesteps', 0): kwargs = processing_correction.correction_callback(p, timestep, kwargs) - # if p.scheduled_prompt and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs: - # try: - # i = (step + 1) % len(p.prompt_embeds) - # kwargs["prompt_embeds"] = p.prompt_embeds[i][0:1].expand(kwargs["prompt_embeds"].shape) - # j = (step + 1) % len(p.negative_embeds) - # kwargs["negative_prompt_embeds"] = p.negative_embeds[j][0:1].expand(kwargs["negative_prompt_embeds"].shape) - # except Exception as e: - # shared.log.debug(f"Callback: {e}") + if p.embedder is not None: + try: + if 'prompt_embeds' in kwargs: + kwargs["prompt_embeds"] = p.embedder("prompt_embeds", step + 1) + if 'negative_prompt_embeds' in kwargs: + kwargs["negative_prompt_embeds"] = p.embedder("negative_prompt_embeds", step + 1) + except Exception as e: + shared.log.debug(f"Callback: {e}") if step == int(getattr(pipe, 'num_timesteps', 100) * p.cfg_end) and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs: if "PAG" in shared.sd_model.__class__.__name__: pipe._guidance_scale = 1.001 if pipe._guidance_scale > 1 else pipe._guidance_scale # pylint: disable=protected-access diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index ecf0cebd5..5ba0e8a74 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -2,8 +2,8 @@ import os import math import time import typing -import torch from collections import OrderedDict +import torch from compel.embeddings_provider import BaseTextualInversionManager, EmbeddingsProvider from transformers import PreTrainedTokenizer from modules import shared, prompt_parser, devices, sd_models @@ -43,16 +43,16 @@ def prepare_model(): class PromptEmbedder: def __init__(self, prompts, negative_prompts, clip_skip, p): t0 = time.time() - # self.prompts, self.negative_prompts, _, _ = fix_prompts(prompts, negative_prompts, None, None) self.prompts = prompts self.negative_prompts = negative_prompts self.batchsize = len(self.prompts) - self.allsame = self.compare_prompts() # collapses batched prompts to single prompt if same + self.allsame = self.compare_prompts() # collapses batched prompts to single prompt if possible self.steps = p.steps self.clip_skip = clip_skip + # All embeds are nested lists, outer list batch length, inner schedule length self.prompt_embeds = [[]] * self.batchsize self.positive_pooleds = [[]] * self.batchsize - self.negative_embeds = [[]] * self.batchsize + self.negative_prompt_embeds = [[]] * self.batchsize self.negative_pooleds = [[]] * self.batchsize self.positive_schedule = None self.negative_schedule = None @@ -68,31 +68,31 @@ class PromptEmbedder: self.scheduled_encode(pipe, batchidx) else: self.encode(pipe, prompt, negative_prompt, batchidx) - if self.allsame: - self.fix_batch_embeds() - debug(f"Prompt encode: time={(time.time() - t0):.3f}") self.checkcache(p) + debug(f"Prompt encode: time={(time.time() - t0):.3f}") def checkcache(self, p): if shared.opts.sd_textencoder_cache_size == 0: return False + def flatten(xss): return [x for xs in xss for x in xs] # unpack EN data in case of TE LoRA en_data = p.extra_network_data en_data = [idx.items for item in en_data.values() for idx in item] - key = str([self.prompts, self.negative_prompts, self.batchsize, self.clip_skip, self.steps, en_data]) + effective_batch = 1 if self.allsame else self.batchsize + key = str([self.prompts, self.negative_prompts, effective_batch, self.clip_skip, self.steps, en_data]) item = cache.get(key) if not item: - if not any([flatten(emb) for emb in [self.prompt_embeds, - self.negative_embeds, - self.positive_pooleds, - self.negative_pooleds]]): + if not any(flatten(emb) for emb in [self.prompt_embeds, + self.negative_prompt_embeds, + self.positive_pooleds, + self.negative_pooleds]): return False else: cache[key] = {'prompt_embeds': self.prompt_embeds, - 'negative_embeds': self.negative_embeds, + 'negative_prompt_embeds': self.negative_prompt_embeds, 'positive_pooleds': self.positive_pooleds, 'negative_pooleds': self.negative_pooleds, } @@ -102,6 +102,11 @@ class PromptEmbedder: if item: self.__dict__.update(cache[key]) cache.move_to_end(key) + if self.allsame and len(self.prompt_embeds) < self.batchsize: # If current batch larger than cached + self.prompt_embeds = [self.prompt_embeds[0]] * self.batchsize + self.positive_pooleds = [self.positive_pooleds[0]] * self.batchsize + self.negative_prompt_embeds = [self.negative_prompt_embeds[0]] * self.batchsize + self.negative_pooleds = [self.negative_pooleds[0]] * self.batchsize debug(f"Prompt cache: Retrieving {key}") return True @@ -119,7 +124,7 @@ class PromptEmbedder: self.scheduled_prompt = scheduled or neg_scheduled def scheduled_encode(self, pipe, batchidx): - prompt_dict = {} + prompt_dict = {} # index cache for i in range(max(len(self.positive_schedule), len(self.negative_schedule))): positive_prompt = self.positive_schedule[i % len(self.positive_schedule)] negative_prompt = self.negative_schedule[i % len(self.negative_schedule)] @@ -131,20 +136,14 @@ class PromptEmbedder: self.encode(pipe, positive_prompt, negative_prompt, batchidx) prompt_dict[positive_prompt+negative_prompt] = i - def extend_embeds(self, batchidx, idx): + def extend_embeds(self, batchidx, idx): # Extends scheduled prompt via index self.prompt_embeds[batchidx].append(self.prompt_embeds[batchidx][idx]) - self.negative_embeds[batchidx].append(self.negative_embeds[batchidx][idx]) + self.negative_prompt_embeds[batchidx].append(self.negative_prompt_embeds[batchidx][idx]) if len(self.positive_pooleds[batchidx]) > 0: self.positive_pooleds[batchidx].append(self.positive_pooleds[batchidx][idx]) if len(self.negative_pooleds[batchidx]) > 0: self.negative_pooleds[batchidx].append(self.negative_pooleds[batchidx][idx]) - def fix_batch_embeds(self): - self.prompt_embeds = self.prompt_embeds[0] * self.batchsize - self.positive_pooleds = self.positive_pooleds[0] * self.batchsize - self.negative_embeds = self.negative_embeds[0] * self.batchsize - self.negative_pooleds = self.negative_pooleds[0] * self.batchsize - def encode(self, pipe, positive_prompt, negative_prompt, batchidx): if shared.opts.prompt_attention == "xhinker parser" or 'Flux' in pipe.__class__.__name__: prompt_embed, positive_pooled, negative_embed, negative_pooled = get_xhinker_text_embeddings( @@ -155,7 +154,7 @@ class PromptEmbedder: if prompt_embed is not None: self.prompt_embeds[batchidx].append(prompt_embed) if negative_embed is not None: - self.negative_embeds[batchidx].append(negative_embed) + self.negative_prompt_embeds[batchidx].append(negative_embed) if positive_pooled is not None: self.positive_pooleds[batchidx].append(positive_pooled) if negative_pooled is not None: @@ -169,14 +168,14 @@ class PromptEmbedder: def __call__(self, key, step=0): batch = getattr(self, key) res = [] - for embed in batch: - if len(embed) == 0: + for i in range(self.batchsize): + if len(batch[i]) == 0: return None - if len(embed) == 1: - res.append(embed[0]) else: - res.append(embed[step]) - return torch.stack(res) + res.append(batch[i][step]) + if step != 0: # For Callback + res.append(batch[i][step]) # Diffusers internally doubles batch dimension + return torch.cat(res) def compel_hijack(self, token_ids: torch.Tensor, attention_mask: typing.Optional[torch.Tensor] = None) -> torch.Tensor: @@ -221,9 +220,9 @@ def insert_parser_highjack(pipename): debug("Load Standard Parser hijack") - insert_parser_highjack("Initialize") + # from https://github.com/damian0815/compel/blob/main/src/compel/diffusers_textual_inversion_manager.py class DiffusersTextualInversionManager(BaseTextualInversionManager): def __init__(self, pipe, tokenizer): @@ -270,12 +269,6 @@ class DiffusersTextualInversionManager(BaseTextualInversionManager): def get_prompt_schedule(prompt, steps): t0 = time.time() - if shared.native: - # TODO prompt scheduling - # prompt schedule returns array of prompts which would require that each prompt is fed to the model per-step - # prompt scheduling should instead interpolate between each prompt in schedule - # this temporarily disables prompt scheduling - return [prompt], False temp = [] schedule = prompt_parser.get_learned_conditioning_prompt_schedules([prompt], steps)[0] if all(x == schedule[0] for x in schedule): @@ -319,118 +312,6 @@ def get_tokens(msg, prompt): debug(f'Prompt tokenizer: type={msg} tokens={token_count} {tokens}') -def encode_prompts(pipe, p, prompts: list, negative_prompts: list, steps: int, clip_skip: typing.Optional[int] = None): - params_match = prompts == cache.get('prompts', None) and negative_prompts == cache.get('negative_prompts', None) and clip_skip == cache.get('clip_skip', None) and steps == cache.get('steps', None) - if ( - 'StableDiffusion' not in pipe.__class__.__name__ and - 'DemoFusion' not in pipe.__class__.__name__ and - 'StableCascade' not in pipe.__class__.__name__ and - 'Flux' not in pipe.__class__.__name__ - ): - shared.log.warning(f"Prompt parser not supported: {pipe.__class__.__name__}") - return - elif shared.opts.sd_textencoder_cache and cache.get('model_type', None) == shared.sd_model_type and params_match: - p.prompt_embeds = cache.get('prompt_embeds', None) - p.positive_pooleds = cache.get('positive_pooleds', None) - p.negative_embeds = cache.get('negative_embeds', None) - p.negative_pooleds = cache.get('negative_pooleds', None) - p.scheduled_prompt = cache.get('scheduled_prompt', None) - debug("Prompt encode: cached") - return - else: - t0 = time.time() - if shared.opts.diffusers_offload_mode == "balanced": - pipe = sd_models.apply_balanced_offload(pipe) - elif hasattr(pipe, "maybe_free_model_hooks"): - pipe.maybe_free_model_hooks() - devices.torch_gc() - - prompt_embeds, positive_pooleds, negative_embeds, negative_pooleds = [], [], [], [] - last_prompt, last_negative = None, None - for prompt, negative in zip(prompts, negative_prompts): - prompt_embed, positive_pooled, negative_embed, negative_pooled = None, None, None, None - if last_prompt == prompt and last_negative == negative: - prompt_embeds.append(prompt_embeds[-1]) - negative_embeds.append(negative_embeds[-1]) - if len(positive_pooleds) > 0: - positive_pooleds.append(positive_pooleds[-1]) - if len(negative_pooleds) > 0: - negative_pooleds.append(negative_pooleds[-1]) - continue - positive_schedule, scheduled = get_prompt_schedule(prompt, steps) - negative_schedule, neg_scheduled = get_prompt_schedule(negative, steps) - p.scheduled_prompt = scheduled or neg_scheduled - p.prompt_embeds = [] - p.positive_pooleds = [] - p.negative_embeds = [] - p.negative_pooleds = [] - - for i in range(max(len(positive_schedule), len(negative_schedule))): - positive_prompt = positive_schedule[i % len(positive_schedule)] - negative_prompt = negative_schedule[i % len(negative_schedule)] - if shared.opts.prompt_attention == "xhinker parser" or 'Flux' in pipe.__class__.__name__: - prompt_embed, positive_pooled, negative_embed, negative_pooled = get_xhinker_text_embeddings(pipe, positive_prompt, negative_prompt, clip_skip) - else: - prompt_embed, positive_pooled, negative_embed, negative_pooled = get_weighted_text_embeddings(pipe, positive_prompt, negative_prompt, clip_skip) - if prompt_embed is not None: - prompt_embeds.append(prompt_embed) - if negative_embed is not None: - negative_embeds.append(negative_embed) - if positive_pooled is not None: - positive_pooleds.append(positive_pooled) - if negative_pooled is not None: - negative_pooleds.append(negative_pooled) - last_prompt, last_negative = prompt, negative - # TODO prompt scheduling - # interpolation should happen here and then we can re-enable prompt scheduling - # ive tried simple torch.mean and its not good-enough - - def fix_length(embeds): - max_len = max([e.shape[1] for e in embeds if e is not None]) - for i, e in enumerate(embeds): - if e is not None and e.shape[1] < max_len: - expanded = torch.zeros((e.shape[0], max_len, e.shape[2]), device=e.device, dtype=e.dtype) - expanded[:, :e.shape[1], :] = e - embeds[i] = expanded - return torch.cat(embeds, dim=0).to(devices.device, dtype=devices.dtype) - - if len(prompt_embeds) > 0: - p.prompt_embeds.append(fix_length(prompt_embeds)) - if len(negative_embeds) > 0: - p.negative_embeds.append(fix_length(negative_embeds)) - if len(positive_pooleds) > 0: - p.positive_pooleds.append(fix_length(positive_pooleds)) - if len(negative_pooleds) > 0: - p.negative_pooleds.append(fix_length(negative_pooleds)) - - if shared.opts.sd_textencoder_cache and p.batch_size == 1: - cache.update({ - 'prompt_embeds': p.prompt_embeds, - 'negative_embeds': p.negative_embeds, - 'positive_pooleds': p.positive_pooleds, - 'negative_pooleds': p.negative_pooleds, - 'scheduled_prompt': p.scheduled_prompt, - 'prompts': prompts, - 'negative_prompts': negative_prompts, - 'clip_skip': clip_skip, - 'steps': steps, - 'model_type': shared.sd_model_type - }) - else: - cache.clear() - if debug_enabled: - get_tokens('positive', prompts[0]) - get_tokens('negative', negative_prompts[0]) - if shared.opts.diffusers_offload_mode == "balanced": - pipe = sd_models.apply_balanced_offload(pipe) - elif hasattr(pipe, "maybe_free_model_hooks"): - # text encoder will stay in the vram and cause oom, send everything back to cpu before continuing - pipe.maybe_free_model_hooks() - debug(f"Prompt encode: time={(time.time() - t0):.3f}") - devices.torch_gc() - return - - def normalize_prompt(pairs: list): num_words = 0 total_weight = 0 @@ -516,6 +397,7 @@ def pad_to_same_length(pipe, embeds, empty_embedding_providers=None): embeds[i] = embed return embeds + def split_prompts(prompt, SD3 = False): if prompt.find("TE2:") != -1: prompt, prompt2 = prompt.split("TE2:") From 38303f0c6138e01f36fc14e000a7916734069210 Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Tue, 29 Oct 2024 21:24:10 -0500 Subject: [PATCH 04/40] Cache unload --- modules/sd_models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index bc293f5fc..11f601260 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1377,8 +1377,9 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.embedding_db.load_textual_inversion_embeddings(force_reload=True) timer.record("embeddings") - from modules.prompt_parser_diffusers import insert_parser_highjack - insert_parser_highjack(sd_model.__class__.__name__) + from modules import prompt_parser_diffusers + prompt_parser_diffusers.insert_parser_highjack(sd_model.__class__.__name__) + prompt_parser_diffusers.cache.clear() set_diffuser_options(sd_model, vae, op, offload=False) if shared.opts.nncf_compress_weights and not ('Model' in shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"): From 3932d5fd1b2f69a119217871bf6b286321fdcac5 Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Wed, 30 Oct 2024 23:24:50 -0500 Subject: [PATCH 05/40] Move embedder object, cleanup stepwise lora --- modules/extra_networks.py | 1 + modules/processing_args.py | 36 +++++++++++++++--------------- modules/processing_callbacks.py | 11 +++++---- modules/prompt_parser_diffusers.py | 5 +++-- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/modules/extra_networks.py b/modules/extra_networks.py index 673549b6b..b464bd349 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -104,6 +104,7 @@ def activate(p, extra_network_data, step=0): p.extra_network_data = extra_network_data if stepwise: + p.stepwise_lora = True shared.opts.data['lora_functional'] = functional diff --git a/modules/processing_args.py b/modules/processing_args.py index 34dd97a1c..5cdf290a7 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -117,7 +117,7 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 'Flux' in model.__class__.__name__ ): try: - p.embedder = prompt_parser_diffusers.PromptEmbedder(prompts, negative_prompts, clip_skip, p) + prompt_parser_diffusers.embedder = prompt_parser_diffusers.PromptEmbedder(prompts, negative_prompts, steps, clip_skip, p) parser = shared.opts.prompt_attention except Exception as e: shared.log.error(f'Prompt parser encode: {e}') @@ -128,27 +128,27 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 if 'prompt' in possible: if 'OmniGen' in model.__class__.__name__: prompts = [p.replace('|image|', '<|image_1|>') for p in prompts] - if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and p.embedder is not None: - args['prompt_embeds'] = p.embedder('prompt_embeds') + if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and prompt_parser_diffusers.embedder is not None: + args['prompt_embeds'] = prompt_parser_diffusers.embedder('prompt_embeds') if 'StableCascade' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0: - args['prompt_embeds_pooled'] = p.embedder('positive_pooleds').unsqueeze(0) - elif 'XL' in model.__class__.__name__ and p.embedder is not None: - args['pooled_prompt_embeds'] = p.embedder('positive_pooleds') - elif 'StableDiffusion3' in model.__class__.__name__ and p.embedder is not None: - args['pooled_prompt_embeds'] = p.embedder('positive_pooleds') - elif 'Flux' in model.__class__.__name__ and p.embedder is not None: - args['pooled_prompt_embeds'] = p.embedder('positive_pooleds') + args['prompt_embeds_pooled'] = prompt_parser_diffusers.embedder('positive_pooleds').unsqueeze(0) + elif 'XL' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None: + args['pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('positive_pooleds') + elif 'StableDiffusion3' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None: + args['pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('positive_pooleds') + elif 'Flux' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None: + args['pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('positive_pooleds') else: args['prompt'] = prompts if 'negative_prompt' in possible: - if hasattr(model, 'text_encoder') and 'negative_prompt_embeds' in possible and p.embedder is not None: - args['negative_prompt_embeds'] = p.embedder('negative_prompt_embeds') - if 'StableCascade' in model.__class__.__name__ and p.embedder is not None: - args['negative_prompt_embeds_pooled'] = p.embedder('negative_pooleds').unsqueeze(0) - if 'XL' in model.__class__.__name__ and p.embedder is not None: - args['negative_pooled_prompt_embeds'] = p.embedder('negative_pooleds') - if 'StableDiffusion3' in model.__class__.__name__ and p.embedder is not None: - args['negative_pooled_prompt_embeds'] = p.embedder('negative_pooleds') + if hasattr(model, 'text_encoder') and 'negative_prompt_embeds' in possible and prompt_parser_diffusers.embedder is not None: + args['negative_prompt_embeds'] = prompt_parser_diffusers.embedder('negative_prompt_embeds') + if 'StableCascade' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None: + args['negative_prompt_embeds_pooled'] = prompt_parser_diffusers.embedder('negative_pooleds').unsqueeze(0) + if 'XL' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None: + args['negative_pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('negative_pooleds') + if 'StableDiffusion3' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None: + args['negative_pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('negative_pooleds') else: if 'PixArtSigmaPipeline' in model.__class__.__name__: # pixart-sigma pipeline throws list-of-list for negative prompt args['negative_prompt'] = negative_prompts[0] diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 5c24aead0..3ace64ed8 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -3,8 +3,7 @@ import os import time import torch import numpy as np -from modules import shared, processing_correction, extra_networks, timer - +from modules import shared, processing_correction, extra_networks, timer, prompt_parser_diffusers p = None debug_callback = shared.log.trace if os.environ.get('SD_CALLBACK_DEBUG', None) is not None else lambda *args, **kwargs: None @@ -49,7 +48,7 @@ def diffusers_callback(pipe, step: int, timestep: int, kwargs: dict): if shared.state.interrupted or shared.state.skipped: raise AssertionError('Interrupted...') time.sleep(0.1) - if hasattr(p, "extra_network_data"): + if hasattr(p, "stepwise_lora"): extra_networks.activate(p, p.extra_network_data, step=step) if latents is None: return kwargs @@ -67,12 +66,12 @@ def diffusers_callback(pipe, step: int, timestep: int, kwargs: dict): pipe.set_ip_adapter_scale(ip_adapter_scales) if step != getattr(pipe, 'num_timesteps', 0): kwargs = processing_correction.correction_callback(p, timestep, kwargs) - if p.embedder is not None: + if prompt_parser_diffusers.embedder is not None: try: if 'prompt_embeds' in kwargs: - kwargs["prompt_embeds"] = p.embedder("prompt_embeds", step + 1) + kwargs["prompt_embeds"] = prompt_parser_diffusers.embedder("prompt_embeds", step + 1) if 'negative_prompt_embeds' in kwargs: - kwargs["negative_prompt_embeds"] = p.embedder("negative_prompt_embeds", step + 1) + kwargs["negative_prompt_embeds"] = prompt_parser_diffusers.embedder("negative_prompt_embeds", step + 1) except Exception as e: shared.log.debug(f"Callback: {e}") if step == int(getattr(pipe, 'num_timesteps', 100) * p.cfg_end) and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs: diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 5ba0e8a74..678649a66 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -16,6 +16,7 @@ orig_encode_token_ids_to_embeddings = EmbeddingsProvider._encode_token_ids_to_em token_dict = None # used by helper get_tokens token_type = None # used by helper get_tokens cache = OrderedDict() +embedder = None def prompt_compatible(): @@ -41,13 +42,13 @@ def prepare_model(): class PromptEmbedder: - def __init__(self, prompts, negative_prompts, clip_skip, p): + def __init__(self, prompts, negative_prompts, steps, clip_skip, p): t0 = time.time() self.prompts = prompts self.negative_prompts = negative_prompts self.batchsize = len(self.prompts) self.allsame = self.compare_prompts() # collapses batched prompts to single prompt if possible - self.steps = p.steps + self.steps = steps self.clip_skip = clip_skip # All embeds are nested lists, outer list batch length, inner schedule length self.prompt_embeds = [[]] * self.batchsize From e06ee1008a769e2a6ddca39078c28bd055a61a78 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 6 Nov 2024 11:05:00 -0500 Subject: [PATCH 06/40] fix check Signed-off-by: Vladimir Mandic --- extensions-builtin/sdnext-modernui | 2 +- modules/processing_args.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 71bdbbd9c..895addec9 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 71bdbbd9c0a55ccea38cbf6fb01483323ac93676 +Subproject commit 895addec9ef65498ed44311d27db0adf699e512d diff --git a/modules/processing_args.py b/modules/processing_args.py index 601615238..67066eedc 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -129,7 +129,7 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 if 'prompt' in possible: if 'OmniGen' in model.__class__.__name__: prompts = [p.replace('|image|', '<|image_1|>') for p in prompts] - if hasattr(model, 'text_encoder') and hasattr(model, 'tokenizer') and 'prompt_embeds' in possible and len(p.prompt_embeds) > 0 and p.prompt_embeds[0] is not None: + if hasattr(model, 'text_encoder') and hasattr(model, 'tokenizer') and 'prompt_embeds' in possible and prompt_parser_diffusers.embedder is not None: args['prompt_embeds'] = prompt_parser_diffusers.embedder('prompt_embeds') if 'StableCascade' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0: args['prompt_embeds_pooled'] = prompt_parser_diffusers.embedder('positive_pooleds').unsqueeze(0) From 34d6d5f92b96b757d94eb2b89fd76efb4ac568e5 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 8 Nov 2024 08:53:42 -0500 Subject: [PATCH 07/40] update Signed-off-by: Vladimir Mandic --- extensions-builtin/sdnext-modernui | 2 +- modules/prompt_parser_diffusers.py | 7 ++++--- wiki | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 895addec9..257be050a 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 895addec9ef65498ed44311d27db0adf699e512d +Subproject commit 257be050afd46a21e77cc9fe60a04d30ed5ffbe4 diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 678649a66..eec6b0d32 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -167,13 +167,14 @@ class PromptEmbedder: pipe = prepare_model() def __call__(self, key, step=0): - batch = getattr(self, key) + batch = getattr(self, key) # for batch-size=1, len(batch)==1 res = [] for i in range(self.batchsize): - if len(batch[i]) == 0: + if len(batch[i]) == 0: # if not using prompt-scheduling, this will be len(batch[i])==1 return None else: - res.append(batch[i][step]) + # causes error in callback + res.append(batch[i][step]) # and this requests element for specific step when called from callback - but self.scheduled_prompt==False so len(batch[i])==1 and step is list index out-of-bounds! if step != 0: # For Callback res.append(batch[i][step]) # Diffusers internally doubles batch dimension return torch.cat(res) diff --git a/wiki b/wiki index 2dba58a69..47ea50e91 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 2dba58a6962b70e92a077dcda8f178f5e811f175 +Subproject commit 47ea50e9152a13325dd1daf92bc50b700783182f From fb4638288b33f7aa47fd1f57ee60c15e3d24e038 Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Fri, 8 Nov 2024 23:31:52 -0600 Subject: [PATCH 08/40] fix IndexError, change callback type --- modules/processing_callbacks.py | 2 +- modules/prompt_parser_diffusers.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 59887c5c4..9e3c0cd31 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -73,7 +73,7 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} if 'negative_prompt_embeds' in kwargs: kwargs["negative_prompt_embeds"] = prompt_parser_diffusers.embedder("negative_prompt_embeds", step + 1) except Exception as e: - shared.log.debug(f"Callback: {e}") + debug_callback(f"Callback: {e}") if step == int(getattr(pipe, 'num_timesteps', 100) * p.cfg_end) and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs: if "PAG" in shared.sd_model.__class__.__name__: pipe._guidance_scale = 1.001 if pipe._guidance_scale > 1 else pipe._guidance_scale # pylint: disable=protected-access diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index eec6b0d32..907cc7208 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -172,11 +172,12 @@ class PromptEmbedder: for i in range(self.batchsize): if len(batch[i]) == 0: # if not using prompt-scheduling, this will be len(batch[i])==1 return None - else: - # causes error in callback + try: res.append(batch[i][step]) # and this requests element for specific step when called from callback - but self.scheduled_prompt==False so len(batch[i])==1 and step is list index out-of-bounds! - if step != 0: # For Callback - res.append(batch[i][step]) # Diffusers internally doubles batch dimension + except IndexError: + res.append(batch[i][0]) + if step != 0: # For Callback + res.append(res[-1]) # Diffusers internally doubles batch dimension return torch.cat(res) From 26359067421c117cbeb1ba28204128ce1db64df7 Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Mon, 11 Nov 2024 18:37:08 -0600 Subject: [PATCH 09/40] fix scheduled prompt because diffusers callback is broken --- modules/processing_callbacks.py | 22 ++++++++++++++-------- modules/prompt_parser_diffusers.py | 10 ++++------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 9e3c0cd31..52ea3e575 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -13,6 +13,19 @@ def set_callbacks_p(processing): global p # pylint: disable=global-statement p = processing +def prompt_callback(step, kwargs): + if prompt_parser_diffusers.embedder is None or 'prompt_embeds' not in kwargs: + return kwargs + try: + prompt_embeds = prompt_parser_diffusers.embedder('prompt_embeds', step + 1) + negative_prompt_embeds = prompt_parser_diffusers.embedder('negative_prompt_embeds', step + 1) + if p.cfg_scale > 1: # Perform guidance + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) # Combined embeds + assert prompt_embeds.shape == kwargs['prompt_embeds'].shape, f"prompt_embed shape mismatch {kwargs['prompt_embeds'].shape} {prompt_embeds.shape}" + kwargs['prompt_embeds'] = prompt_embeds + except Exception as e: + debug_callback(f"Callback: {e}") + return kwargs def diffusers_callback_legacy(step: int, timestep: int, latents: typing.Union[torch.FloatTensor, np.ndarray]): if p is None: @@ -66,14 +79,7 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} pipe.set_ip_adapter_scale(ip_adapter_scales) if step != getattr(pipe, 'num_timesteps', 0): kwargs = processing_correction.correction_callback(p, timestep, kwargs) - if prompt_parser_diffusers.embedder is not None: - try: - if 'prompt_embeds' in kwargs: - kwargs["prompt_embeds"] = prompt_parser_diffusers.embedder("prompt_embeds", step + 1) - if 'negative_prompt_embeds' in kwargs: - kwargs["negative_prompt_embeds"] = prompt_parser_diffusers.embedder("negative_prompt_embeds", step + 1) - except Exception as e: - debug_callback(f"Callback: {e}") + kwargs = prompt_callback(step, kwargs) # monkey patch for diffusers callback issues if step == int(getattr(pipe, 'num_timesteps', 100) * p.cfg_end) and 'prompt_embeds' in kwargs and 'negative_prompt_embeds' in kwargs: if "PAG" in shared.sd_model.__class__.__name__: pipe._guidance_scale = 1.001 if pipe._guidance_scale > 1 else pipe._guidance_scale # pylint: disable=protected-access diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 907cc7208..0e10e5fe1 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -167,17 +167,15 @@ class PromptEmbedder: pipe = prepare_model() def __call__(self, key, step=0): - batch = getattr(self, key) # for batch-size=1, len(batch)==1 + batch = getattr(self, key) res = [] for i in range(self.batchsize): - if len(batch[i]) == 0: # if not using prompt-scheduling, this will be len(batch[i])==1 + if len(batch[i]) == 0: # if asking for a null key, ie pooled on SD1.5 return None try: - res.append(batch[i][step]) # and this requests element for specific step when called from callback - but self.scheduled_prompt==False so len(batch[i])==1 and step is list index out-of-bounds! + res.append(batch[i][step]) except IndexError: - res.append(batch[i][0]) - if step != 0: # For Callback - res.append(res[-1]) # Diffusers internally doubles batch dimension + res.append(batch[i][0]) # if not scheduled, return default return torch.cat(res) From 66820edb63858885561aa7d1d2d24a92f3e76615 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 12 Nov 2024 11:02:34 -0500 Subject: [PATCH 10/40] update Signed-off-by: Vladimir Mandic --- modules/face/instantid.py | 2 +- modules/face/photomaker.py | 2 +- modules/processing_args.py | 9 +++-- modules/processing_info.py | 17 +++++---- modules/prompt_parser.py | 16 ++++---- modules/prompt_parser_diffusers.py | 41 ++++++++++++++------- modules/prompt_parser_xhinker.py | 2 +- modules/shared.py | 59 ++++++++++++++++++++++++------ scripts/animatediff.py | 2 +- scripts/ctrlx.py | 2 +- scripts/ledits.py | 2 +- scripts/mixture_tiling.py | 2 +- scripts/mulan.py | 2 +- scripts/regional_prompting.py | 2 +- scripts/x_adapter.py | 2 +- scripts/xyz_grid_classes.py | 4 ++ wiki | 2 +- 17 files changed, 113 insertions(+), 55 deletions(-) diff --git a/modules/face/instantid.py b/modules/face/instantid.py index 9c7c16f61..662d17c7d 100644 --- a/modules/face/instantid.py +++ b/modules/face/instantid.py @@ -68,7 +68,7 @@ def instant_id(p: processing.StableDiffusionProcessing, app, source_images, stre processing.process_init(p) p.init(p.all_prompts, p.all_seeds, p.all_subseeds) orig_prompt_attention = shared.opts.prompt_attention - shared.opts.data['prompt_attention'] = 'Fixed attention' # otherwise need to deal with class_tokens_mask + shared.opts.data['prompt_attention'] = 'fixed' # otherwise need to deal with class_tokens_mask p.task_args['image_embeds'] = face_embeds[0].shape # placeholder p.task_args['image'] = face_images[0] p.task_args['controlnet_conditioning_scale'] = float(conditioning) diff --git a/modules/face/photomaker.py b/modules/face/photomaker.py index c8f58b42a..b89f28a10 100644 --- a/modules/face/photomaker.py +++ b/modules/face/photomaker.py @@ -49,7 +49,7 @@ def photo_maker(p: processing.StableDiffusionProcessing, input_images, trigger, shared.sd_model.to(dtype=devices.dtype) orig_prompt_attention = shared.opts.prompt_attention - shared.opts.data['prompt_attention'] = 'Fixed attention' # otherwise need to deal with class_tokens_mask + shared.opts.data['prompt_attention'] = 'fixed' # otherwise need to deal with class_tokens_mask p.task_args['input_id_images'] = input_images p.task_args['start_merge_step'] = int(start * p.steps) p.task_args['prompt'] = p.all_prompts[0] if p.all_prompts is not None else p.prompt diff --git a/modules/processing_args.py b/modules/processing_args.py index 67066eedc..4cd12a04d 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -107,12 +107,11 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 debug(f'Diffusers pipeline possible: {possible}') prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2) - parser = 'Fixed attention' steps = kwargs.get("num_inference_steps", None) or len(getattr(p, 'timesteps', ['1'])) clip_skip = kwargs.pop("clip_skip", 1) - # prompt_parser_diffusers.fix_position_ids(model) - if shared.opts.prompt_attention != 'Fixed attention' and 'Onnx' not in model.__class__.__name__ and ( + parser = 'fixed' + if shared.opts.prompt_attention != 'fixed' and 'Onnx' not in model.__class__.__name__ and ( 'StableDiffusion' in model.__class__.__name__ or 'StableCascade' in model.__class__.__name__ or 'Flux' in model.__class__.__name__ @@ -125,6 +124,8 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 if os.environ.get('SD_PROMPT_DEBUG', None) is not None: errors.display(e, 'Prompt parser encode') timer.process.record('encode', reset=False) + else: + prompt_parser_diffusers.embedder = None if 'prompt' in possible: if 'OmniGen' in model.__class__.__name__: @@ -156,7 +157,7 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 else: args['negative_prompt'] = negative_prompts - if 'clip_skip' in possible and parser == 'Fixed attention': + if 'clip_skip' in possible and parser == 'fixed': if clip_skip == 1: pass # clip_skip = None else: diff --git a/modules/processing_info.py b/modules/processing_info.py index e798211b1..f721b3717 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -4,6 +4,7 @@ from modules import shared, sd_samplers_common, sd_vae, generation_parameters_co from modules.processing_class import StableDiffusionProcessing +debug = shared.log.trace if os.environ.get('SD_PROCESS_DEBUG', None) is not None else lambda *args, **kwargs: None if not shared.native: from modules import sd_hijack else: @@ -39,27 +40,27 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No ops.reverse() args = { # basic + "Size": f"{p.width}x{p.height}" if hasattr(p, 'width') and hasattr(p, 'height') else None, + "Sampler": p.sampler_name if p.sampler_name != 'Default' else None, "Steps": p.steps, "Seed": all_seeds[index], - "Sampler": p.sampler_name if p.sampler_name != 'Default' else None, + "Seed resize from": None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}", "CFG scale": p.cfg_scale if p.cfg_scale > 1.0 else None, "CFG end": p.cfg_end if p.cfg_end < 1.0 else None, - "Size": f"{p.width}x{p.height}" if hasattr(p, 'width') and hasattr(p, 'height') else None, + "Clip skip": p.clip_skip if p.clip_skip > 1 else None, "Batch": f'{p.n_iter}x{p.batch_size}' if p.n_iter > 1 or p.batch_size > 1 else None, - "Parser": shared.opts.prompt_attention.split()[0], "Model": None if (not shared.opts.add_model_name_to_info) or (not shared.sd_model.sd_checkpoint_info.model_name) else shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', ''), "Model hash": getattr(p, 'sd_model_hash', None if (not shared.opts.add_model_hash_to_info) or (not shared.sd_model.sd_model_hash) else shared.sd_model.sd_model_hash), "VAE": (None if not shared.opts.add_model_name_to_info or sd_vae.loaded_vae_file is None else os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0]) if p.full_quality else 'TAESD', - "Seed resize from": None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}", - "Clip skip": p.clip_skip if p.clip_skip > 1 else None, "Prompt2": p.refiner_prompt if len(p.refiner_prompt) > 0 else None, "Negative2": p.refiner_negative if len(p.refiner_negative) > 0 else None, "Styles": "; ".join(p.styles) if p.styles is not None and len(p.styles) > 0 else None, - "Tiling": p.tiling if p.tiling else None, # sdnext - "Backend": 'Diffusers' if shared.native else 'Original', "App": 'SD.Next', "Version": git_commit, + "Backend": 'Diffusers' if shared.native else 'Original', + "Pipeline": 'LDM', + "Parser": shared.opts.prompt_attention.split()[0], "Comment": comment, "Operations": '; '.join(ops).replace('"', '') if len(p.ops) > 0 else 'none', } @@ -165,7 +166,9 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No if isinstance(v, str): if len(v) == 0 or v == '0x0': del args[k] + debug(f'Infotext: args={args}') params_text = ", ".join([k if k == v else f'{k}: {generation_parameters_copypaste.quote(v)}' for k, v in args.items()]) negative_prompt_text = f"\nNegative prompt: {all_negative_prompts[index]}" if all_negative_prompts[index] else "" infotext = f"{all_prompts[index]}{negative_prompt_text}\n{params_text}".strip() + debug(f'Infotext: "{infotext}"') return infotext diff --git a/modules/prompt_parser.py b/modules/prompt_parser.py index 2a71d5053..3a1288097 100644 --- a/modules/prompt_parser.py +++ b/modules/prompt_parser.py @@ -308,11 +308,11 @@ def parse_prompt_attention(text): res = [] round_brackets = [] square_brackets = [] - if opts.prompt_attention == 'Fixed attention': + if opts.prompt_attention == 'fixed': res = [[text, 1.0]] debug(f'Prompt: parser="{opts.prompt_attention}" {res}') return res - elif opts.prompt_attention == 'Compel parser': + elif opts.prompt_attention == 'compel': conjunction = Compel.parse_prompt_string(text) if conjunction is None or conjunction.prompts is None or conjunction.prompts is None or len(conjunction.prompts[0].children) == 0: return [["", 1.0]] @@ -321,7 +321,7 @@ def parse_prompt_attention(text): res.append([frag.text, frag.weight]) debug(f'Prompt: parser="{opts.prompt_attention}" {res}') return res - elif opts.prompt_attention == 'A1111 parser': + elif opts.prompt_attention == 'a1111': re_attention = re_attention_v1 whitespace = '' else: @@ -360,7 +360,7 @@ def parse_prompt_attention(text): for i, part in enumerate(parts): if i > 0: res.append(["BREAK", -1]) - if opts.prompt_attention == 'Full parser': + if opts.prompt_attention == 'native': part = re_clean.sub("", part) part = re_whitespace.sub(" ", part).strip() if len(part) == 0: @@ -392,15 +392,15 @@ if __name__ == "__main__": log.info(f'Schedules: {all_schedules}') for schedule in all_schedules: log.info(f'Schedule: {schedule[0]}') - opts.data['prompt_attention'] = 'Fixed attention' + opts.data['prompt_attention'] = 'fixed' output_list = parse_prompt_attention(schedule[1]) log.info(f' Fixed: {output_list}') - opts.data['prompt_attention'] = 'Compel parser' + opts.data['prompt_attention'] = 'compel' output_list = parse_prompt_attention(schedule[1]) log.info(f' Compel: {output_list}') - opts.data['prompt_attention'] = 'A1111 parser' + opts.data['prompt_attention'] = 'a1111' output_list = parse_prompt_attention(schedule[1]) log.info(f' A1111: {output_list}') - opts.data['prompt_attention'] = 'Full parser' + opts.data['prompt_attention'] = 'native' log.info = parse_prompt_attention(schedule[1]) log.info(f' Full: {output_list}') diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 0e10e5fe1..e53af7957 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -47,6 +47,7 @@ class PromptEmbedder: self.prompts = prompts self.negative_prompts = negative_prompts self.batchsize = len(self.prompts) + self.attention = None self.allsame = self.compare_prompts() # collapses batched prompts to single prompt if possible self.steps = steps self.clip_skip = clip_skip @@ -75,6 +76,10 @@ class PromptEmbedder: def checkcache(self, p): if shared.opts.sd_textencoder_cache_size == 0: return False + if self.attention != shared.opts.prompt_attention: + debug(f"Prompt change: parser={shared.opts.prompt_attention}") + cache.clear() + return False def flatten(xss): return [x for xs in xss for x in xs] @@ -97,23 +102,22 @@ class PromptEmbedder: 'positive_pooleds': self.positive_pooleds, 'negative_pooleds': self.negative_pooleds, } - debug(f"Prompt cache: Adding {key}") + debug(f"Prompt cache: add={key}") while len(cache) > int(shared.opts.sd_textencoder_cache_size): cache.popitem(last=False) if item: self.__dict__.update(cache[key]) cache.move_to_end(key) - if self.allsame and len(self.prompt_embeds) < self.batchsize: # If current batch larger than cached + if self.allsame and len(self.prompt_embeds) < self.batchsize: self.prompt_embeds = [self.prompt_embeds[0]] * self.batchsize self.positive_pooleds = [self.positive_pooleds[0]] * self.batchsize self.negative_prompt_embeds = [self.negative_prompt_embeds[0]] * self.batchsize self.negative_pooleds = [self.negative_pooleds[0]] * self.batchsize - debug(f"Prompt cache: Retrieving {key}") + debug(f"Prompt cache: get={key}") return True def compare_prompts(self): - same = (self.prompts == [self.prompts[0]] * len(self.prompts) and - self.negative_prompts == [self.negative_prompts[0]] * len(self.negative_prompts)) + same = (self.prompts == [self.prompts[0]] * len(self.prompts) and self.negative_prompts == [self.negative_prompts[0]] * len(self.negative_prompts)) if same: self.prompts = [self.prompts[0]] self.negative_prompts = [self.negative_prompts[0]] @@ -123,6 +127,7 @@ class PromptEmbedder: self.positive_schedule, scheduled = get_prompt_schedule(prompt, self.steps) self.negative_schedule, neg_scheduled = get_prompt_schedule(negative_prompt, self.steps) self.scheduled_prompt = scheduled or neg_scheduled + debug(f"Prompt schedule: positive={self.positive_schedule} negative={self.negative_schedule} scheduled={scheduled}") def scheduled_encode(self, pipe, batchidx): prompt_dict = {} # index cache @@ -138,20 +143,21 @@ class PromptEmbedder: prompt_dict[positive_prompt+negative_prompt] = i def extend_embeds(self, batchidx, idx): # Extends scheduled prompt via index - self.prompt_embeds[batchidx].append(self.prompt_embeds[batchidx][idx]) - self.negative_prompt_embeds[batchidx].append(self.negative_prompt_embeds[batchidx][idx]) + if len(self.prompt_embeds[batchidx]) > 0: + self.prompt_embeds[batchidx].append(self.prompt_embeds[batchidx][idx]) + if len(self.negative_prompt_embeds[batchidx]) > 0: + self.negative_prompt_embeds[batchidx].append(self.negative_prompt_embeds[batchidx][idx]) if len(self.positive_pooleds[batchidx]) > 0: self.positive_pooleds[batchidx].append(self.positive_pooleds[batchidx][idx]) if len(self.negative_pooleds[batchidx]) > 0: self.negative_pooleds[batchidx].append(self.negative_pooleds[batchidx][idx]) def encode(self, pipe, positive_prompt, negative_prompt, batchidx): - if shared.opts.prompt_attention == "xhinker parser" or 'Flux' in pipe.__class__.__name__: - prompt_embed, positive_pooled, negative_embed, negative_pooled = get_xhinker_text_embeddings( - pipe, positive_prompt, negative_prompt, self.clip_skip) + self.attention = shared.opts.prompt_attention + if self.attention == "xhinker" or 'Flux' in pipe.__class__.__name__: + prompt_embed, positive_pooled, negative_embed, negative_pooled = get_xhinker_text_embeddings(pipe, positive_prompt, negative_prompt, self.clip_skip) else: - prompt_embed, positive_pooled, negative_embed, negative_pooled = get_weighted_text_embeddings( - pipe, positive_prompt, negative_prompt, self.clip_skip) + prompt_embed, positive_pooled, negative_embed, negative_pooled = get_weighted_text_embeddings(pipe, positive_prompt, negative_prompt, self.clip_skip) if prompt_embed is not None: self.prompt_embeds[batchidx].append(prompt_embed) if negative_embed is not None: @@ -311,6 +317,7 @@ def get_tokens(msg, prompt): tokens.append(f'UNK_{i}') token_count = len(ids) - int(has_bos_token) - int(has_eos_token) debug(f'Prompt tokenizer: type={msg} tokens={token_count} {tokens}') + return token_count def normalize_prompt(pairs: list): @@ -338,6 +345,12 @@ def get_prompts_with_weights(prompt: str): if shared.opts.prompt_mean_norm: texts_and_weights = normalize_prompt(texts_and_weights) texts, text_weights = zip(*texts_and_weights) + if debug_enabled: + all_tokens = 0 + for text in texts: + tokens = get_tokens('section', text) + all_tokens += tokens + debug(f'Prompt tokenizer: parser={shared.opts.prompt_attention} tokens={all_tokens}') debug(f'Prompt: weights={texts_and_weights} time={(time.time() - t0):.3f}') return texts, text_weights @@ -479,7 +492,7 @@ def get_weighted_text_embeddings(pipe, prompt: str = "", neg_prompt: str = "", c # negative prompt has no keywords embed, ntokens = embedding_providers[i].get_embeddings_for_weighted_prompt_fragments(text_batch=[negatives[i]], fragment_weights_batch=[negative_weights[i]], device=device, should_return_tokens=True) negative_prompt_embeds.append(embed) - debug(f'Prompt: unpadded shape={prompt_embeds[0].shape} TE{i+1} ptokens={torch.count_nonzero(ptokens)} ntokens={torch.count_nonzero(ntokens)} time={(time.time() - t0):.3f}') + debug(f'Prompt: unpadded={prompt_embeds[0].shape} TE{i+1} ptokens={torch.count_nonzero(ptokens)} ntokens={torch.count_nonzero(ntokens)} time={(time.time() - t0):.3f}') if SD3: t0 = time.time() pooled_prompt_embeds.append(embedding_providers[0].get_pooled_embeddings(texts=positives[0] if len(positives[0]) == 1 else [" ".join(positives[0])], device=device)) @@ -488,7 +501,7 @@ def get_weighted_text_embeddings(pipe, prompt: str = "", neg_prompt: str = "", c negative_pooled_prompt_embeds.append(embedding_providers[1].get_pooled_embeddings(texts=negatives[-1] if len(negatives[-1]) == 1 else [" ".join(negatives[-1])], device=device)) pooled_prompt_embeds = torch.cat(pooled_prompt_embeds, dim=-1) negative_pooled_prompt_embeds = torch.cat(negative_pooled_prompt_embeds, dim=-1) - debug(f'Prompt: pooled shape={pooled_prompt_embeds[0].shape} time={(time.time() - t0):.3f}') + debug(f'Prompt: pooled={pooled_prompt_embeds[0].shape} time={(time.time() - t0):.3f}') elif prompt_embeds[-1].shape[-1] > 768: t0 = time.time() if shared.opts.diffusers_pooled == "weighted": diff --git a/modules/prompt_parser_xhinker.py b/modules/prompt_parser_xhinker.py index 6a8acf8c6..c0ddc9bc7 100644 --- a/modules/prompt_parser_xhinker.py +++ b/modules/prompt_parser_xhinker.py @@ -1305,7 +1305,7 @@ def get_weighted_text_embeddings_sd3( # ---------------------- get neg t5 embeddings ------------------------- neg_prompt_tokens_3 = torch.tensor([neg_prompt_tokens_3], dtype=torch.long) - t5_neg_prompt_embeds = pipe.text_encoder_3(neg_prompt_tokens_3.to(pipe.pipe.text_encoder_3.device))[0].squeeze(0) + t5_neg_prompt_embeds = pipe.text_encoder_3(neg_prompt_tokens_3.to(pipe.text_encoder_3.device))[0].squeeze(0) t5_neg_prompt_embeds = t5_neg_prompt_embeds.to(device=pipe.text_encoder_3.device) # add weight to neg t5 embeddings diff --git a/modules/shared.py b/modules/shared.py index 867651e7a..171a777e9 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -273,6 +273,40 @@ class OptionInfo: self.comment_after += " (requires restart)" return self + def validate(self, opt, value): + args = self.component_args if self.component_args is not None else {} + if callable(args): + try: + args = args() + except Exception: + args = {} + choices = args.get("choices", []) + if callable(choices): + try: + choices = choices() + except Exception: + choices = [] + if len(choices) > 0: + if not isinstance(value, list): + value = [value] + for v in value: + if v not in choices: + log.warning(f'Setting validation: "{opt}"="{v}" default="{self.default}" choices={choices}') + return False + minimum = args.get("minimum", None) + maximum = args.get("maximum", None) + if (minimum is not None and value < minimum) or (maximum is not None and value > maximum): + log.error(f'Setting validation: "{opt}"={value} default={self.default} minimum={minimum} maximum={maximum}') + return False + return True + + def __str__(self) -> str: + args = self.component_args if self.component_args is not None else {} + if callable(args): + args = args() + choices = args.get("choices", []) + return f'OptionInfo: label="{self.label}" section="{self.section}" component="{self.component}" default="{self.default}" refresh="{self.refresh is not None}" change="{self.onchange is not None}" args={args} choices={choices}' + def options_section(section_identifier, options_dict): for v in options_dict.values(): @@ -442,7 +476,7 @@ options_templates.update(options_section(('sd', "Execution & Models"), { "model_reuse_dict": OptionInfo(False, "Reuse loaded model dictionary", gr.Checkbox, {"visible": False}), "prompt_mean_norm": OptionInfo(False, "Prompt attention normalization", gr.Checkbox), "comma_padding_backtrack": OptionInfo(20, "Prompt padding", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1, "visible": not native }), - "prompt_attention": OptionInfo("Full parser", "Prompt attention parser", gr.Radio, {"choices": ["Full parser", "Compel parser", "xhinker parser", "A1111 parser", "Fixed attention"] }), + "prompt_attention": OptionInfo("native", "Prompt attention parser", gr.Radio, {"choices": ["native", "compel", "xhinker", "a1111", "fixed"] }), "latent_history": OptionInfo(16, "Latent history size", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}), "sd_checkpoint_cache": OptionInfo(0, "Cached models", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1, "visible": not native }), "sd_vae_checkpoint_cache": OptionInfo(0, "Cached VAEs", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1, "visible": False}), @@ -994,7 +1028,7 @@ class Options: if filename is None: filename = self.filename if cmd_opts.freeze: - log.warning(f'Settings saving is disabled: {filename}') + log.warning(f'Setting: fn="{filename}" save disabled') return try: # output = json.dumps(self.data, indent=2) @@ -1002,12 +1036,12 @@ class Options: unused_settings = [] if os.environ.get('SD_CONFIG_DEBUG', None) is not None: - log.debug('Config: user settings') + log.debug('Settings: user') for k, v in self.data.items(): log.trace(f' Config: item={k} value={v} default={self.data_labels[k].default if k in self.data_labels else None}') - log.debug('Config: default settings') + log.debug('Settings: defaults') for k in self.data_labels.keys(): - log.trace(f' Config: item={k} default={self.data_labels[k].default}') + log.trace(f' Setting: item={k} default={self.data_labels[k].default}') for k, v in self.data.items(): if k in self.data_labels: @@ -1022,9 +1056,9 @@ class Options: unused_settings.append(k) writefile(diff, filename, silent=silent) if len(unused_settings) > 0: - log.debug(f"Unused settings: {unused_settings}") + log.debug(f"Settings: unused={unused_settings}") except Exception as err: - log.error(f'Save settings failed: {filename} {err}') + log.error(f'Settings: fn="{filename}" {err}') def save(self, filename=None, silent=False): threading.Thread(target=self.save_atomic, args=(filename, silent)).start() @@ -1040,7 +1074,7 @@ class Options: if filename is None: filename = self.filename if not os.path.isfile(filename): - log.debug(f'Created default config: {filename}') + log.debug(f'Settings: fn="{filename}" created') self.save(filename) return self.data = readfile(filename, lock=True) @@ -1048,13 +1082,16 @@ class Options: self.data['quicksettings_list'] = [i.strip() for i in self.data.get('quicksettings').split(',')] unknown_settings = [] for k, v in self.data.items(): - info = self.data_labels.get(k, None) + info: OptionInfo = self.data_labels.get(k, None) + if not info.validate(k, v): + self.data[k] = info.default if info is not None and not self.same_type(info.default, v): - log.error(f"Error: bad setting value: {k}: {v} ({type(v).__name__}; expected {type(info.default).__name__})") + log.warning(f"Setting validation: {k}={v} ({type(v).__name__} expected={type(info.default).__name__})") + self.data[k] = info.default if info is None and k not in compatibility_opts and not k.startswith('uiux_'): unknown_settings.append(k) if len(unknown_settings) > 0: - log.debug(f"Unknown settings: {unknown_settings}") + log.warning(f"Setting validation: unknown={unknown_settings}") def onchange(self, key, func, call=True): item = self.data_labels.get(key) diff --git a/scripts/animatediff.py b/scripts/animatediff.py index 09f9e33a9..fca09424b 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -258,7 +258,7 @@ class Script(scripts.Script): shared.log.debug(f'AnimateDiff args: {p.task_args}') set_prompt(p) orig_prompt_attention = shared.opts.prompt_attention - shared.opts.data['prompt_attention'] = 'Fixed attention' + shared.opts.data['prompt_attention'] = 'fixed' processed: processing.Processed = processing.process_images(p) # runs processing using main loop shared.opts.data['prompt_attention'] = orig_prompt_attention devices.torch_gc() diff --git a/scripts/ctrlx.py b/scripts/ctrlx.py index acfdd5d6e..69d5994df 100644 --- a/scripts/ctrlx.py +++ b/scripts/ctrlx.py @@ -49,7 +49,7 @@ class Script(scripts.Script): from modules.ctrlx.utils import get_self_recurrence_schedule orig_prompt_attention = shared.opts.prompt_attention - shared.opts.data['prompt_attention'] = 'Fixed attention' + shared.opts.data['prompt_attention'] = 'fixed' shared.sd_model = sd_models.switch_pipe(CtrlXStableDiffusionXLPipeline, shared.sd_model) shared.sd_model.restore_pipeline = self.restore diff --git a/scripts/ledits.py b/scripts/ledits.py index 1a0e929f0..ba9d49f89 100644 --- a/scripts/ledits.py +++ b/scripts/ledits.py @@ -44,7 +44,7 @@ class Script(scripts.Script): orig_offload = shared.opts.diffusers_model_cpu_offload orig_prompt_attention = shared.opts.prompt_attention shared.opts.data['diffusers_model_cpu_offload'] = False - shared.opts.data['prompt_attention'] = 'Fixed attention' + shared.opts.data['prompt_attention'] = 'fixed' # shared.sd_model.maybe_free_model_hooks() # ledits is not compatible with offloading # shared.sd_model.has_accelerate = False sd_models.move_model(shared.sd_model, devices.device, force=True) diff --git a/scripts/mixture_tiling.py b/scripts/mixture_tiling.py index 5dcaf0156..13e48ce11 100644 --- a/scripts/mixture_tiling.py +++ b/scripts/mixture_tiling.py @@ -66,7 +66,7 @@ class Script(scripts.Script): shared.sd_model = orig_pipeline return sd_models.set_diffuser_options(shared.sd_model) - shared.opts.data['prompt_attention'] = 'Fixed attention' # this pipeline is not compatible with embeds + shared.opts.data['prompt_attention'] = 'fixed' # this pipeline is not compatible with embeds shared.sd_model.to(torch.float32) # this pipeline unet is not compatible with fp16 processing.fix_seed(p) # set pipeline specific params, note that standard params are applied when applicable diff --git a/scripts/mulan.py b/scripts/mulan.py index 4b80a7c87..c2ad10d2e 100644 --- a/scripts/mulan.py +++ b/scripts/mulan.py @@ -87,7 +87,7 @@ class Script(scripts.Script): # mulan only works with single image, single prompt and in fixed attention p.batch_size = 1 p.n_iter = 1 - shared.opts.prompt_attention = 'Fixed attention' + shared.opts.prompt_attention = 'fixed' if isinstance(p.prompt, list): p.prompt = p.prompt[0] p.task_args['prompt'] = p.prompt diff --git a/scripts/regional_prompting.py b/scripts/regional_prompting.py index cecef747d..08b84dd94 100644 --- a/scripts/regional_prompting.py +++ b/scripts/regional_prompting.py @@ -64,7 +64,7 @@ class Script(scripts.Script): shared.sd_model = orig_pipeline return sd_models.set_diffuser_options(shared.sd_model) - shared.opts.data['prompt_attention'] = 'Fixed attention' # this pipeline is not compatible with embeds + shared.opts.data['prompt_attention'] = 'fixed' # this pipeline is not compatible with embeds processing.fix_seed(p) # set pipeline specific params, note that standard params are applied when applicable rp_args = { diff --git a/scripts/x_adapter.py b/scripts/x_adapter.py index 553a20d30..c67eca18b 100644 --- a/scripts/x_adapter.py +++ b/scripts/x_adapter.py @@ -107,7 +107,7 @@ class Script(scripts.Script): pipe.to(device=devices.device, dtype=devices.dtype) except Exception: pass - shared.opts.data['prompt_attention'] = 'Fixed attention' + shared.opts.data['prompt_attention'] = 'fixed' prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) negative = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) p.task_args['prompt'] = prompt diff --git a/scripts/xyz_grid_classes.py b/scripts/xyz_grid_classes.py index 4898c6b73..08ea279f4 100644 --- a/scripts/xyz_grid_classes.py +++ b/scripts/xyz_grid_classes.py @@ -37,6 +37,7 @@ class SharedSettingsStackHelper(object): sd_text_encoder = None extra_networks_default_multiplier = None disable_weights_auto_swap = None + prompt_attention = None def __enter__(self): #Save overridden settings so they can be restored later. @@ -52,6 +53,7 @@ class SharedSettingsStackHelper(object): self.sd_text_encoder = shared.opts.sd_text_encoder self.extra_networks_default_multiplier = shared.opts.extra_networks_default_multiplier self.disable_weights_auto_swap = shared.opts.disable_weights_auto_swap + self.prompt_attention = shared.opts.prompt_attention shared.opts.data["disable_weights_auto_swap"] = False def __exit__(self, exc_type, exc_value, tb): @@ -62,6 +64,7 @@ class SharedSettingsStackHelper(object): shared.opts.data["tome_ratio"] = self.tome_ratio shared.opts.data["todo_ratio"] = self.todo_ratio shared.opts.data["extra_networks_default_multiplier"] = self.extra_networks_default_multiplier + shared.opts.data["prompt_attention"] = self.prompt_attention if self.sd_model_checkpoint != shared.opts.sd_model_checkpoint: shared.opts.data["sd_model_checkpoint"] = self.sd_model_checkpoint sd_models.reload_model_weights(op='model') @@ -92,6 +95,7 @@ axis_options = [ AxisOption("[Model] Dictionary", str, apply_dict, fmt=format_value_add_label, cost=0.9, choices=lambda: ['None'] + list(sd_models.checkpoints_list)), AxisOption("[Prompt] Search & replace", str, apply_prompt, fmt=format_value_add_label), AxisOption("[Prompt] Prompt order", str_permutations, apply_order, fmt=format_value_join_list), + AxisOption("[Prompt] Prompt parser", str, apply_setting("prompt_attention"), choices=lambda: ["native", "compel", "xhinker", "a1111", "fixed"]), AxisOption("[Network] LoRA", str, apply_lora, cost=0.5, choices=list_lora), AxisOption("[Network] LoRA strength", float, apply_setting('extra_networks_default_multiplier')), AxisOption("[Network] Styles", str, apply_styles, choices=lambda: [s.name for s in shared.prompt_styles.styles.values()]), diff --git a/wiki b/wiki index 47ea50e91..352fc655b 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 47ea50e9152a13325dd1daf92bc50b700783182f +Subproject commit 352fc655b0dc9edb22aac093186da087ba18b474 From b42e9253e34dd1e267193eb8a6d0ea177295e02a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 6 Nov 2024 21:21:12 -0500 Subject: [PATCH 11/40] pullid offload compatibility and extra samplers Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/pulid/__init__.py | 1 + modules/pulid/pulid_sampling.py | 571 ++++++++++++++++++++++++++++++++ modules/pulid/pulid_sdxl.py | 7 +- modules/pulid/pulid_utils.py | 176 ---------- modules/sd_models.py | 15 +- scripts/pulid_ext.py | 12 +- 7 files changed, 595 insertions(+), 188 deletions(-) create mode 100644 modules/pulid/pulid_sampling.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2726a1864..4e08cb147 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ This release can be considered an LTS release before we kick off the next round - select in *scripts -> pulid* - compatible with *sdxl* - can be used in xyz grid + - *note*: this module contains several advanced features on top of original implementation - [InstantIR](https://github.com/instantX-research/InstantIR): Blind Image Restoration with Instant Generative Reference - alternative to traditional `img2img` with more control over restoration process - select in *image -> scripts -> instantir* diff --git a/modules/pulid/__init__.py b/modules/pulid/__init__.py index 000f45293..785b849c2 100644 --- a/modules/pulid/__init__.py +++ b/modules/pulid/__init__.py @@ -8,3 +8,4 @@ sys.path.append(os.path.dirname(__file__)) from pulid_sdxl import StableDiffusionXLPuLIDPipeline from pulid_utils import resize_numpy_image_long as resize import attention_processor as attention +import pulid_sampling as sampling diff --git a/modules/pulid/pulid_sampling.py b/modules/pulid/pulid_sampling.py new file mode 100644 index 000000000..9996f035a --- /dev/null +++ b/modules/pulid/pulid_sampling.py @@ -0,0 +1,571 @@ +import math +from scipy import integrate +import torch +from torch import nn +from torchdiffeq import odeint +import torchsde +from tqdm.auto import trange + + +def append_zero(x): + return torch.cat([x, x.new_zeros([1])]) + + +def get_sigmas_karras(n, sigma_min, sigma_max, rho=7., device='cpu'): + """Constructs the noise schedule of Karras et al. (2022).""" + ramp = torch.linspace(0, 1, n) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return append_zero(sigmas).to(device) + + +def get_sigmas_exponential(n, sigma_min, sigma_max, device='cpu'): + """Constructs an exponential noise schedule.""" + sigmas = torch.linspace(math.log(sigma_max), math.log(sigma_min), n, device=device).exp() + return append_zero(sigmas) + + +def get_sigmas_polyexponential(n, sigma_min, sigma_max, rho=1., device='cpu'): + """Constructs an polynomial in log sigma noise schedule.""" + ramp = torch.linspace(1, 0, n, device=device) ** rho + sigmas = torch.exp(ramp * (math.log(sigma_max) - math.log(sigma_min)) + math.log(sigma_min)) + return append_zero(sigmas) + + +def get_sigmas_vp(n, beta_d=19.9, beta_min=0.1, eps_s=1e-3, device='cpu'): + """Constructs a continuous VP noise schedule.""" + t = torch.linspace(1, eps_s, n, device=device) + sigmas = torch.sqrt(torch.exp(beta_d * t ** 2 / 2 + beta_min * t) - 1) + return append_zero(sigmas) + + +def append_dims(x, target_dims): + """Appends dimensions to the end of a tensor until it has target_dims dimensions.""" + dims_to_append = target_dims - x.ndim + if dims_to_append < 0: + raise ValueError(f'input has {x.ndim} dims but target_dims is {target_dims}, which is less') + return x[(...,) + (None,) * dims_to_append] + + +def to_d(x, sigma, denoised): + """Converts a denoiser output to a Karras ODE derivative.""" + return (x - denoised) / append_dims(sigma, x.ndim) + + +def get_ancestral_step(sigma_from, sigma_to, eta=1.): + """Calculates the noise level (sigma_down) to step down to and the amount + of noise to add (sigma_up) when doing an ancestral sampling step.""" + if not eta: + return sigma_to, 0. + sigma_up = min(sigma_to, eta * (sigma_to ** 2 * (sigma_from ** 2 - sigma_to ** 2) / sigma_from ** 2) ** 0.5) + sigma_down = (sigma_to ** 2 - sigma_up ** 2) ** 0.5 + return sigma_down, sigma_up + + +def default_noise_sampler(x): + return lambda sigma, sigma_next: torch.randn_like(x) + + +class BatchedBrownianTree: + """A wrapper around torchsde.BrownianTree that enables batches of entropy.""" + + def __init__(self, x, t0, t1, seed=None, **kwargs): + t0, t1, self.sign = self.sort(t0, t1) + w0 = kwargs.get('w0', torch.zeros_like(x)) + if seed is None: + seed = torch.randint(0, 2 ** 63 - 1, []).item() + self.batched = True + try: + assert len(seed) == x.shape[0] + w0 = w0[0] + except TypeError: + seed = [seed] + self.batched = False + self.trees = [torchsde.BrownianTree(t0, w0, t1, entropy=s, **kwargs) for s in seed] + + @staticmethod + def sort(a, b): + return (a, b, 1) if a < b else (b, a, -1) + + def __call__(self, t0, t1): + t0, t1, sign = self.sort(t0, t1) + w = torch.stack([tree(t0, t1) for tree in self.trees]) * (self.sign * sign) + return w if self.batched else w[0] + + +class BrownianTreeNoiseSampler: + """A noise sampler backed by a torchsde.BrownianTree. + + Args: + x (Tensor): The tensor whose shape, device and dtype to use to generate + random samples. + sigma_min (float): The low end of the valid interval. + sigma_max (float): The high end of the valid interval. + seed (int or List[int]): The random seed. If a list of seeds is + supplied instead of a single integer, then the noise sampler will + use one BrownianTree per batch item, each with its own seed. + transform (callable): A function that maps sigma to the sampler's + internal timestep. + """ + + def __init__(self, x, sigma_min, sigma_max, seed=None, transform=lambda x: x): + self.transform = transform + t0, t1 = self.transform(torch.as_tensor(sigma_min)), self.transform(torch.as_tensor(sigma_max)) + self.tree = BatchedBrownianTree(x, t0, t1, seed) + + def __call__(self, sigma, sigma_next): + t0, t1 = self.transform(torch.as_tensor(sigma)), self.transform(torch.as_tensor(sigma_next)) + return self.tree(t0, t1) / (t1 - t0).abs().sqrt() + + +@torch.no_grad() +def sample_euler(model, x, sigmas, extra_args=None, callback=None, disable=None, s_churn=0., s_tmin=0., s_tmax=float('inf'), s_noise=1.): + """Implements Algorithm 2 (Euler steps) from Karras et al. (2022).""" + extra_args = {} if extra_args is None else extra_args + s_in = x.new_ones([x.shape[0]]) + for i in trange(len(sigmas) - 1, disable=disable): + gamma = min(s_churn / (len(sigmas) - 1), 2 ** 0.5 - 1) if s_tmin <= sigmas[i] <= s_tmax else 0. + eps = torch.randn_like(x) * s_noise + sigma_hat = sigmas[i] * (gamma + 1) + if gamma > 0: + x = x + eps * (sigma_hat ** 2 - sigmas[i] ** 2) ** 0.5 + denoised = model(x, sigma_hat * s_in, **extra_args) + d = to_d(x, sigma_hat, denoised) + if callback is not None: + callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigma_hat, 'denoised': denoised}) + dt = sigmas[i + 1] - sigma_hat + # Euler method + x = x + (d * dt).to(x.dtype) + return x + + +@torch.no_grad() +def sample_euler_ancestral(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None): + """Ancestral sampling with Euler method steps.""" + extra_args = {} if extra_args is None else extra_args + noise_sampler = default_noise_sampler(x) if noise_sampler is None else noise_sampler + s_in = x.new_ones([x.shape[0]]) + for i in trange(len(sigmas) - 1, disable=disable): + denoised = model(x, sigmas[i] * s_in, **extra_args) + sigma_down, sigma_up = get_ancestral_step(sigmas[i], sigmas[i + 1], eta=eta) + if callback is not None: + callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) + d = to_d(x, sigmas[i], denoised) + # Euler method + dt = sigma_down - sigmas[i] + x = x + (d * dt).to(x.dtype) + if sigmas[i + 1] > 0: + x = x + (noise_sampler(sigmas[i], sigmas[i + 1]) * s_noise * sigma_up).to(x.dtype) + return x + + +def linear_multistep_coeff(order, t, i, j): + if order - 1 > i: + raise ValueError(f'Order {order} too high for step {i}') + def fn(tau): + prod = 1. + for k in range(order): + if j == k: + continue + prod *= (tau - t[i - k]) / (t[i - j] - t[i - k]) + return prod + return integrate.quad(fn, t[i], t[i + 1], epsrel=1e-4)[0] + + +@torch.no_grad() +def log_likelihood(model, x, sigma_min, sigma_max, extra_args=None, atol=1e-4, rtol=1e-4): + extra_args = {} if extra_args is None else extra_args + s_in = x.new_ones([x.shape[0]]) + v = torch.randint_like(x, 2) * 2 - 1 + fevals = 0 + def ode_fn(sigma, x): + nonlocal fevals + with torch.enable_grad(): + x = x[0].detach().requires_grad_() + denoised = model(x, sigma * s_in, **extra_args) + d = to_d(x, sigma, denoised) + fevals += 1 + grad = torch.autograd.grad((d * v).sum(), x)[0] + d_ll = (v * grad).flatten(1).sum(1) + return d.detach(), d_ll + x_min = x, x.new_zeros([x.shape[0]]) + t = x.new_tensor([sigma_min, sigma_max]) + sol = odeint(ode_fn, x_min, t, atol=atol, rtol=rtol, method='dopri5') + latent, delta_ll = sol[0][-1], sol[1][-1] + ll_prior = torch.distributions.Normal(0, sigma_max).log_prob(latent).flatten(1).sum(1) + return ll_prior + delta_ll, {'fevals': fevals} + + +class PIDStepSizeController: + """A PID controller for ODE adaptive step size control.""" + def __init__(self, h, pcoeff, icoeff, dcoeff, order=1, accept_safety=0.81, eps=1e-8): + self.h = h + self.b1 = (pcoeff + icoeff + dcoeff) / order + self.b2 = -(pcoeff + 2 * dcoeff) / order + self.b3 = dcoeff / order + self.accept_safety = accept_safety + self.eps = eps + self.errs = [] + + def limiter(self, x): + return 1 + math.atan(x - 1) + + def propose_step(self, error): + inv_error = 1 / (float(error) + self.eps) + if not self.errs: + self.errs = [inv_error, inv_error, inv_error] + self.errs[0] = inv_error + factor = self.errs[0] ** self.b1 * self.errs[1] ** self.b2 * self.errs[2] ** self.b3 + factor = self.limiter(factor) + accept = factor >= self.accept_safety + if accept: + self.errs[2] = self.errs[1] + self.errs[1] = self.errs[0] + self.h *= factor + return accept + + +class DPMSolver(nn.Module): + """DPM-Solver. See https://arxiv.org/abs/2206.00927.""" + + def __init__(self, model, extra_args=None, eps_callback=None, info_callback=None): + super().__init__() + self.model = model + self.extra_args = {} if extra_args is None else extra_args + self.eps_callback = eps_callback + self.info_callback = info_callback + + def t(self, sigma): + return -sigma.log() + + def sigma(self, t): + return t.neg().exp() + + def eps(self, eps_cache, key, x, t, *args, **kwargs): + if key in eps_cache: + return eps_cache[key], eps_cache + sigma = self.sigma(t) * x.new_ones([x.shape[0]]) + eps = (x - self.model(x, sigma, *args, **self.extra_args, **kwargs)) / self.sigma(t) + if self.eps_callback is not None: + self.eps_callback() + return eps, {key: eps, **eps_cache} + + def dpm_solver_1_step(self, x, t, t_next, eps_cache=None): + eps_cache = {} if eps_cache is None else eps_cache + h = t_next - t + eps, eps_cache = self.eps(eps_cache, 'eps', x, t) + x_1 = x - self.sigma(t_next) * h.expm1() * eps + return x_1, eps_cache + + def dpm_solver_2_step(self, x, t, t_next, r1=1 / 2, eps_cache=None): + eps_cache = {} if eps_cache is None else eps_cache + h = t_next - t + eps, eps_cache = self.eps(eps_cache, 'eps', x, t) + s1 = t + r1 * h + u1 = x - self.sigma(s1) * (r1 * h).expm1() * eps + eps_r1, eps_cache = self.eps(eps_cache, 'eps_r1', u1, s1) + x_2 = x - self.sigma(t_next) * h.expm1() * eps - self.sigma(t_next) / (2 * r1) * h.expm1() * (eps_r1 - eps) + return x_2, eps_cache + + def dpm_solver_3_step(self, x, t, t_next, r1=1 / 3, r2=2 / 3, eps_cache=None): + eps_cache = {} if eps_cache is None else eps_cache + h = t_next - t + eps, eps_cache = self.eps(eps_cache, 'eps', x, t) + s1 = t + r1 * h + s2 = t + r2 * h + u1 = x - self.sigma(s1) * (r1 * h).expm1() * eps + eps_r1, eps_cache = self.eps(eps_cache, 'eps_r1', u1, s1) + u2 = x - self.sigma(s2) * (r2 * h).expm1() * eps - self.sigma(s2) * (r2 / r1) * ((r2 * h).expm1() / (r2 * h) - 1) * (eps_r1 - eps) + eps_r2, eps_cache = self.eps(eps_cache, 'eps_r2', u2, s2) + x_3 = x - self.sigma(t_next) * h.expm1() * eps - self.sigma(t_next) / r2 * (h.expm1() / h - 1) * (eps_r2 - eps) + return x_3, eps_cache + + def dpm_solver_fast(self, x, t_start, t_end, nfe, eta=0., s_noise=1., noise_sampler=None): + noise_sampler = default_noise_sampler(x) if noise_sampler is None else noise_sampler + if not t_end > t_start and eta: + raise ValueError('eta must be 0 for reverse sampling') + + m = math.floor(nfe / 3) + 1 + ts = torch.linspace(t_start, t_end, m + 1, device=x.device) + + if nfe % 3 == 0: + orders = [3] * (m - 2) + [2, 1] + else: + orders = [3] * (m - 1) + [nfe % 3] + + for i in range(len(orders)): + eps_cache = {} + t, t_next = ts[i], ts[i + 1] + if eta: + sd, su = get_ancestral_step(self.sigma(t), self.sigma(t_next), eta) + t_next_ = torch.minimum(t_end, self.t(sd)) + su = (self.sigma(t_next) ** 2 - self.sigma(t_next_) ** 2) ** 0.5 + else: + t_next_, su = t_next, 0. + + eps, eps_cache = self.eps(eps_cache, 'eps', x, t) + denoised = x - self.sigma(t) * eps + if self.info_callback is not None: + self.info_callback({'x': x, 'i': i, 't': ts[i], 't_up': t, 'denoised': denoised}) + + if orders[i] == 1: + x, eps_cache = self.dpm_solver_1_step(x, t, t_next_, eps_cache=eps_cache) + elif orders[i] == 2: + x, eps_cache = self.dpm_solver_2_step(x, t, t_next_, eps_cache=eps_cache) + else: + x, eps_cache = self.dpm_solver_3_step(x, t, t_next_, eps_cache=eps_cache) + + x = x + su * s_noise * noise_sampler(self.sigma(t), self.sigma(t_next)) + + return x + + def dpm_solver_adaptive(self, x, t_start, t_end, order=3, rtol=0.05, atol=0.0078, h_init=0.05, pcoeff=0., icoeff=1., dcoeff=0., accept_safety=0.81, eta=0., s_noise=1., noise_sampler=None): + noise_sampler = default_noise_sampler(x) if noise_sampler is None else noise_sampler + if order not in {2, 3}: + raise ValueError('order should be 2 or 3') + forward = t_end > t_start + if not forward and eta: + raise ValueError('eta must be 0 for reverse sampling') + h_init = abs(h_init) * (1 if forward else -1) + atol = torch.tensor(atol) + rtol = torch.tensor(rtol) + s = t_start + x_prev = x + accept = True + pid = PIDStepSizeController(h_init, pcoeff, icoeff, dcoeff, 1.5 if eta else order, accept_safety) + info = {'steps': 0, 'nfe': 0, 'n_accept': 0, 'n_reject': 0} + + while s < t_end - 1e-5 if forward else s > t_end + 1e-5: + eps_cache = {} + t = torch.minimum(t_end, s + pid.h) if forward else torch.maximum(t_end, s + pid.h) + if eta: + sd, su = get_ancestral_step(self.sigma(s), self.sigma(t), eta) + t_ = torch.minimum(t_end, self.t(sd)) + su = (self.sigma(t) ** 2 - self.sigma(t_) ** 2) ** 0.5 + else: + t_, su = t, 0. + + eps, eps_cache = self.eps(eps_cache, 'eps', x, s) + denoised = x - self.sigma(s) * eps + + if order == 2: + x_low, eps_cache = self.dpm_solver_1_step(x, s, t_, eps_cache=eps_cache) + x_high, eps_cache = self.dpm_solver_2_step(x, s, t_, eps_cache=eps_cache) + else: + x_low, eps_cache = self.dpm_solver_2_step(x, s, t_, r1=1 / 3, eps_cache=eps_cache) + x_high, eps_cache = self.dpm_solver_3_step(x, s, t_, eps_cache=eps_cache) + delta = torch.maximum(atol, rtol * torch.maximum(x_low.abs(), x_prev.abs())) + error = torch.linalg.norm((x_low - x_high) / delta) / x.numel() ** 0.5 + accept = pid.propose_step(error) + if accept: + x_prev = x_low + x = x_high + su * s_noise * noise_sampler(self.sigma(s), self.sigma(t)) + s = t + info['n_accept'] += 1 + else: + info['n_reject'] += 1 + info['nfe'] += order + info['steps'] += 1 + + if self.info_callback is not None: + self.info_callback({'x': x, 'i': info['steps'] - 1, 't': s, 't_up': s, 'denoised': denoised, 'error': error, 'h': pid.h, **info}) + + return x, info + + +@torch.no_grad() +def sample_dpmpp_2s_ancestral(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None): + """Ancestral sampling with DPM-Solver++(2S) second-order steps.""" + extra_args = {} if extra_args is None else extra_args + noise_sampler = default_noise_sampler(x) if noise_sampler is None else noise_sampler + s_in = x.new_ones([x.shape[0]]) + sigma_fn = lambda t: t.neg().exp() + t_fn = lambda sigma: sigma.log().neg() + + for i in trange(len(sigmas) - 1, disable=disable): + denoised = model(x, sigmas[i] * s_in, **extra_args) + sigma_down, sigma_up = get_ancestral_step(sigmas[i], sigmas[i + 1], eta=eta) + if callback is not None: + callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) + if sigma_down == 0: + # Euler method + d = to_d(x, sigmas[i], denoised) + dt = sigma_down - sigmas[i] + x = x + d * dt + else: + # DPM-Solver++(2S) + t, t_next = t_fn(sigmas[i]), t_fn(sigma_down) + r = 1 / 2 + h = t_next - t + s = t + r * h + x_2 = (sigma_fn(s) / sigma_fn(t)) * x - (-h * r).expm1() * denoised + denoised_2 = model(x_2, sigma_fn(s) * s_in, **extra_args) + x = (sigma_fn(t_next) / sigma_fn(t)) * x - (-h).expm1() * denoised_2 + # Noise addition + if sigmas[i + 1] > 0: + x = x + noise_sampler(sigmas[i], sigmas[i + 1]) * s_noise * sigma_up + return x + + +@torch.no_grad() +def sample_dpmpp_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, r=1 / 2): + """DPM-Solver++ (stochastic).""" + sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max() + noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max) if noise_sampler is None else noise_sampler + extra_args = {} if extra_args is None else extra_args + s_in = x.new_ones([x.shape[0]]) + sigma_fn = lambda t: t.neg().exp() + t_fn = lambda sigma: sigma.log().neg() + + for i in trange(len(sigmas) - 1, disable=disable): + denoised = model(x, sigmas[i] * s_in, **extra_args) + if callback is not None: + callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) + if sigmas[i + 1] == 0: + # Euler method + d = to_d(x, sigmas[i], denoised) + dt = sigmas[i + 1] - sigmas[i] + x = x + d * dt + else: + # DPM-Solver++ + t, t_next = t_fn(sigmas[i]), t_fn(sigmas[i + 1]) + h = t_next - t + s = t + h * r + fac = 1 / (2 * r) + + # Step 1 + sd, su = get_ancestral_step(sigma_fn(t), sigma_fn(s), eta) + s_ = t_fn(sd) + x_2 = (sigma_fn(s_) / sigma_fn(t)) * x - (t - s_).expm1() * denoised + x_2 = x_2 + noise_sampler(sigma_fn(t), sigma_fn(s)) * s_noise * su + denoised_2 = model(x_2, sigma_fn(s) * s_in, **extra_args) + + # Step 2 + sd, su = get_ancestral_step(sigma_fn(t), sigma_fn(t_next), eta) + t_next_ = t_fn(sd) + denoised_d = (1 - fac) * denoised + fac * denoised_2 + x = (sigma_fn(t_next_) / sigma_fn(t)) * x - (t - t_next_).expm1() * denoised_d + x = x + noise_sampler(sigma_fn(t), sigma_fn(t_next)) * s_noise * su + return x + + +@torch.no_grad() +def sample_dpmpp_2m(model, x, sigmas, extra_args=None, callback=None, disable=None): + """DPM-Solver++(2M).""" + extra_args = {} if extra_args is None else extra_args + s_in = x.new_ones([x.shape[0]]) + sigma_fn = lambda t: t.neg().exp() + t_fn = lambda sigma: sigma.log().neg() + old_denoised = None + + for i in trange(len(sigmas) - 1, disable=disable): + denoised = model(x, sigmas[i] * s_in, **extra_args) + if callback is not None: + callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) + t, t_next = t_fn(sigmas[i]), t_fn(sigmas[i + 1]) + h = t_next - t + if old_denoised is None or sigmas[i + 1] == 0: + x = (sigma_fn(t_next) / sigma_fn(t)) * x - (-h).expm1() * denoised + else: + h_last = t - t_fn(sigmas[i - 1]) + r = h_last / h + denoised_d = (1 + 1 / (2 * r)) * denoised - (1 / (2 * r)) * old_denoised + x = (sigma_fn(t_next) / sigma_fn(t)) * x - (-h).expm1() * denoised_d + old_denoised = denoised + return x + + +@torch.no_grad() +def sample_dpmpp_2m_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, solver_type='midpoint'): + """DPM-Solver++(2M) SDE.""" + + if solver_type not in {'heun', 'midpoint'}: + raise ValueError('solver_type must be \'heun\' or \'midpoint\'') + + sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max() + noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max) if noise_sampler is None else noise_sampler + extra_args = {} if extra_args is None else extra_args + s_in = x.new_ones([x.shape[0]]) + + old_denoised = None + h_last = None + + for i in trange(len(sigmas) - 1, disable=disable): + denoised = model(x, sigmas[i] * s_in, **extra_args) + if callback is not None: + callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) + if sigmas[i + 1] == 0: + # Denoising step + x = denoised + else: + # DPM-Solver++(2M) SDE + t, s = -sigmas[i].log(), -sigmas[i + 1].log() + h = s - t + eta_h = eta * h + + x = sigmas[i + 1] / sigmas[i] * (-eta_h).exp() * x + (-h - eta_h).expm1().neg() * denoised + + if old_denoised is not None: + r = h_last / h + if solver_type == 'heun': + x = x + ((-h - eta_h).expm1().neg() / (-h - eta_h) + 1) * (1 / r) * (denoised - old_denoised) + elif solver_type == 'midpoint': + x = x + 0.5 * (-h - eta_h).expm1().neg() * (1 / r) * (denoised - old_denoised) + + if eta: + x = x + noise_sampler(sigmas[i], sigmas[i + 1]) * sigmas[i + 1] * (-2 * eta_h).expm1().neg().sqrt() * s_noise + + old_denoised = denoised + h_last = h + return x + + +@torch.no_grad() +def sample_dpmpp_3m_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None): + """DPM-Solver++(3M) SDE.""" + + sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max() + noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max) if noise_sampler is None else noise_sampler + extra_args = {} if extra_args is None else extra_args + s_in = x.new_ones([x.shape[0]]) + + denoised_1, denoised_2 = None, None + h_1, h_2 = None, None + + for i in trange(len(sigmas) - 1, disable=disable): + denoised = model(x, sigmas[i] * s_in, **extra_args) + if callback is not None: + callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) + if sigmas[i + 1] == 0: + # Denoising step + x = denoised + else: + t, s = -sigmas[i].log(), -sigmas[i + 1].log() + h = s - t + h_eta = h * (eta + 1) + + x = torch.exp(-h_eta) * x + (-h_eta).expm1().neg() * denoised + + if h_2 is not None: + r0 = h_1 / h + r1 = h_2 / h + d1_0 = (denoised - denoised_1) / r0 + d1_1 = (denoised_1 - denoised_2) / r1 + d1 = d1_0 + (d1_0 - d1_1) * r0 / (r0 + r1) + d2 = (d1_0 - d1_1) / (r0 + r1) + phi_2 = h_eta.neg().expm1() / h_eta + 1 + phi_3 = phi_2 / h_eta - 0.5 + x = x + phi_2 * d1 - phi_3 * d2 + elif h_1 is not None: + r = h_1 / h + d = (denoised - denoised_1) / r + phi_2 = h_eta.neg().expm1() / h_eta + 1 + x = x + phi_2 * d + + if eta: + x = x + noise_sampler(sigmas[i], sigmas[i + 1]) * sigmas[i + 1] * (-2 * h * eta).expm1().neg().sqrt() * s_noise + + denoised_1, denoised_2 = denoised, denoised_1 + h_1, h_2 = h, h_1 + return x diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index 0651efab4..de650b839 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -19,13 +19,12 @@ from insightface.app import FaceAnalysis from eva_clip import create_model_and_transforms from eva_clip.constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD from encoders_transformer import IDFormer -from pulid_utils import sample_dpmpp_2m, sample_dpmpp_sde from attention_processor import AttnProcessor2_0 as AttnProcessor from attention_processor import IDAttnProcessor2_0 as IDAttnProcessor class StableDiffusionXLPuLIDPipeline: - def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, sampler='dpmpp_sde', cache_dir=None): + def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, sampler=None, cache_dir=None): super().__init__() self.device = device self.pipe = pipe @@ -90,12 +89,16 @@ class StableDiffusionXLPuLIDPipeline: self.log_sigmas = self.sigmas.log() self.sigma_data = 1.0 + if sampler is not None: + self.sampler = sampler + """ if sampler == 'dpmpp_sde': self.sampler = sample_dpmpp_sde elif sampler == 'dpmpp_2m': self.sampler = sample_dpmpp_2m else: raise NotImplementedError(f'sampler {sampler} not implemented') + """ @property def sigma_min(self): diff --git a/modules/pulid/pulid_utils.py b/modules/pulid/pulid_utils.py index 1a8d3ff06..fd7338b5e 100644 --- a/modules/pulid/pulid_utils.py +++ b/modules/pulid/pulid_utils.py @@ -6,10 +6,7 @@ import random import cv2 import numpy as np import torch -import torch.nn.functional as F -import torchsde from torchvision.utils import make_grid -from tqdm.auto import trange from transformers import PretrainedConfig @@ -21,10 +18,6 @@ def seed_everything(seed): torch.cuda.manual_seed_all(seed) -def is_torch2_available(): - return hasattr(F, "scaled_dot_product_attention") - - def instantiate_from_config(config): if "target" not in config: if config == '__is_first_stage__' or config == "__is_unconditional__": @@ -166,172 +159,3 @@ def tensor2img(tensor, rgb2bgr=True, out_type=np.uint8, min_max=(0, 1)): if len(result) == 1: result = result[0] return result - - -# We didn't find a correct configuration to make the diffusers scheduler align with dpm++2m (karras) in ComfyUI, -# so we copied the ComfyUI code directly. - - -def append_dims(x, target_dims): - """Appends dimensions to the end of a tensor until it has target_dims dimensions.""" - dims_to_append = target_dims - x.ndim - if dims_to_append < 0: - raise ValueError(f'input has {x.ndim} dims but target_dims is {target_dims}, which is less') - expanded = x[(...,) + (None,) * dims_to_append] - # MPS will get inf values if it tries to index into the new axes, but detaching fixes this. - # https://github.com/pytorch/pytorch/issues/84364 - return expanded.detach().clone() if expanded.device.type == 'mps' else expanded - - -def to_d(x, sigma, denoised): - """Converts a denoiser output to a Karras ODE derivative.""" - return (x - denoised) / append_dims(sigma, x.ndim) - - -def get_ancestral_step(sigma_from, sigma_to, eta=1.0): - """Calculates the noise level (sigma_down) to step down to and the amount - of noise to add (sigma_up) when doing an ancestral sampling step.""" - if not eta: - return sigma_to, 0.0 - sigma_up = min(sigma_to, eta * (sigma_to**2 * (sigma_from**2 - sigma_to**2) / sigma_from**2) ** 0.5) - sigma_down = (sigma_to**2 - sigma_up**2) ** 0.5 - return sigma_down, sigma_up - - -class BatchedBrownianTree: - """A wrapper around torchsde.BrownianTree that enables batches of entropy.""" - - def __init__(self, x, t0, t1, seed=None, **kwargs): - self.cpu_tree = True - if "cpu" in kwargs: - self.cpu_tree = kwargs.pop("cpu") - t0, t1, self.sign = self.sort(t0, t1) - w0 = kwargs.get('w0', torch.zeros_like(x)) - if seed is None: - seed = torch.randint(0, 2**63 - 1, []).item() - self.batched = True - try: - assert len(seed) == x.shape[0] - w0 = w0[0] - except TypeError: - seed = [seed] - self.batched = False - if self.cpu_tree: - self.trees = [torchsde.BrownianTree(t0.cpu(), w0.cpu(), t1.cpu(), entropy=s, **kwargs) for s in seed] - else: - self.trees = [torchsde.BrownianTree(t0, w0, t1, entropy=s, **kwargs) for s in seed] - - @staticmethod - def sort(a, b): - return (a, b, 1) if a < b else (b, a, -1) - - def __call__(self, t0, t1): - t0, t1, sign = self.sort(t0, t1) - if self.cpu_tree: - w = torch.stack( - [tree(t0.cpu().float(), t1.cpu().float()).to(t0.dtype).to(t0.device) for tree in self.trees] - ) * (self.sign * sign) - else: - w = torch.stack([tree(t0, t1) for tree in self.trees]) * (self.sign * sign) - - return w if self.batched else w[0] - - -class BrownianTreeNoiseSampler: - """A noise sampler backed by a torchsde.BrownianTree. - - Args: - x (Tensor): The tensor whose shape, device and dtype to use to generate - random samples. - sigma_min (float): The low end of the valid interval. - sigma_max (float): The high end of the valid interval. - seed (int or List[int]): The random seed. If a list of seeds is - supplied instead of a single integer, then the noise sampler will - use one BrownianTree per batch item, each with its own seed. - transform (callable): A function that maps sigma to the sampler's - internal timestep. - """ - - def __init__(self, x, sigma_min, sigma_max, seed=None, transform=lambda x: x, cpu=False): - self.transform = transform - t0, t1 = self.transform(torch.as_tensor(sigma_min)), self.transform(torch.as_tensor(sigma_max)) - self.tree = BatchedBrownianTree(x, t0, t1, seed, cpu=cpu) - - def __call__(self, sigma, sigma_next): - t0, t1 = self.transform(torch.as_tensor(sigma)), self.transform(torch.as_tensor(sigma_next)) - return self.tree(t0, t1) / (t1 - t0).abs().sqrt() - - -@torch.no_grad() -def sample_dpmpp_2m(model, x, sigmas, extra_args=None, callback=None, disable=None): - """DPM-Solver++(2M).""" - extra_args = {} if extra_args is None else extra_args - s_in = x.new_ones([x.shape[0]]) - sigma_fn = lambda t: t.neg().exp() # pylint: disable=unnecessary-lambda-assignment - t_fn = lambda sigma: sigma.log().neg() # pylint: disable=unnecessary-lambda-assignment - old_denoised = None - - for i in trange(len(sigmas) - 1, disable=disable): - denoised = model(x, sigmas[i] * s_in, **extra_args) - if callback is not None: - callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) - t, t_next = t_fn(sigmas[i]), t_fn(sigmas[i + 1]) - h = t_next - t - if old_denoised is None or sigmas[i + 1] == 0: - x = (sigma_fn(t_next) / sigma_fn(t)) * x - (-h).expm1() * denoised - else: - h_last = t - t_fn(sigmas[i - 1]) - r = h_last / h - denoised_d = (1 + 1 / (2 * r)) * denoised - (1 / (2 * r)) * old_denoised - x = (sigma_fn(t_next) / sigma_fn(t)) * x - (-h).expm1() * denoised_d - old_denoised = denoised - return x - - -@torch.no_grad() -def sample_dpmpp_sde( - model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1.0, s_noise=1.0, noise_sampler=None, r=1 / 2 -): - """DPM-Solver++ (stochastic).""" - sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max() - seed = extra_args.get("seed", None) - noise_sampler = ( - BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=seed, cpu=False) - if noise_sampler is None - else noise_sampler - ) - extra_args = {} if extra_args is None else extra_args - s_in = x.new_ones([x.shape[0]]) - sigma_fn = lambda t: t.neg().exp() # pylint: disable=unnecessary-lambda-assignment - t_fn = lambda sigma: sigma.log().neg() # pylint: disable=unnecessary-lambda-assignment - - for i in trange(len(sigmas) - 1, disable=disable): - denoised = model(x, sigmas[i] * s_in, **extra_args) - if callback is not None: - callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) - if sigmas[i + 1] == 0: - # Euler method - d = to_d(x, sigmas[i], denoised) - dt = sigmas[i + 1] - sigmas[i] - x = x + d * dt - else: - # DPM-Solver++ - t, t_next = t_fn(sigmas[i]), t_fn(sigmas[i + 1]) - h = t_next - t - s = t + h * r - fac = 1 / (2 * r) - - # Step 1 - sd, su = get_ancestral_step(sigma_fn(t), sigma_fn(s), eta) - s_ = t_fn(sd) - x_2 = (sigma_fn(s_) / sigma_fn(t)) * x - (t - s_).expm1() * denoised - x_2 = x_2 + noise_sampler(sigma_fn(t), sigma_fn(s)) * s_noise * su - denoised_2 = model(x_2, sigma_fn(s) * s_in, **extra_args) - - # Step 2 - sd, su = get_ancestral_step(sigma_fn(t), sigma_fn(t_next), eta) - t_next_ = t_fn(sd) - denoised_d = (1 - fac) * denoised + fac * denoised_2 - x = (sigma_fn(t_next_) / sigma_fn(t)) * x - (t - t_next_).expm1() * denoised_d - x = x + noise_sampler(sigma_fn(t), sigma_fn(t_next)) * s_noise * su - return x diff --git a/modules/sd_models.py b/modules/sd_models.py index 3ca081d6c..9dd87cec1 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -291,10 +291,13 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model', offload=True): def set_accelerate_to_module(model): - for k in model._internal_dict.keys(): # pylint: disable=protected-access - component = getattr(model, k, None) - if isinstance(component, torch.nn.Module): - component.has_accelerate = True + if hasattr(model, "pipe"): + set_accelerate_to_module(model.pipe) + if hasattr(model, "_internal_dict"): + for k in model._internal_dict.keys(): # pylint: disable=protected-access + component = getattr(model, k, None) + if isinstance(component, torch.nn.Module): + component.has_accelerate = True def set_accelerate(sd_model): @@ -397,6 +400,10 @@ def apply_balanced_offload(sd_model): return module def apply_balanced_offload_to_module(pipe): + if hasattr(pipe, "pipe"): + apply_balanced_offload_to_module(pipe.pipe) + if not hasattr(pipe, "_internal_dict"): + return for module_name in pipe._internal_dict.keys(): # pylint: disable=protected-access module = getattr(pipe, module_name, None) if isinstance(module, torch.nn.Module): diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index d31c18164..54189cc05 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -30,8 +30,6 @@ class Script(scripts.Script): install('insightface', 'insightface', ignore=False) install('albumentations==1.4.3', 'albumentations', ignore=False, reinstall=True) install('pydantic==1.10.15', 'pydantic', ignore=False, reinstall=True) - # if not installed('apex', reload=False, quiet=True): - # install('apex', 'apex', ignore=False) def register(self): # register xyz grid elements def apply_field(field): @@ -74,8 +72,8 @@ class Script(scripts.Script): strength = gr.Slider(label = 'Strength', value = 0.8, mininimum = 0, maximum = 1, step = 0.01) zero = gr.Slider(label = 'Zero', value = 20, mininimum = 0, maximum = 80, step = 1) with gr.Row(): - sampler = gr.Dropdown(label="Sampler", choices=['dpmpp_sde', 'dpmpp_2m'], value='dpmpp_sde', visible=True) - ortho = gr.Dropdown(label="Ortho", choices=['off', 'v1', 'v2'], value='v2', visible=True) + sampler = gr.Dropdown(label="Sampler", value='dpmpp_sde', choices=['dpmpp_2m', 'dpmpp_2m_sde', 'dpmpp_2s_ancestral', 'dpmpp_3m_sde', 'dpmpp_sde', 'euler', 'euler_ancestral']) + ortho = gr.Dropdown(label="Ortho", choices=['off', 'v1', 'v2'], value='v2') with gr.Row(): files = gr.File(label='Input images', file_count='multiple', file_types=['image'], type='file', interactive=True, height=100) with gr.Row(): @@ -124,16 +122,17 @@ class Script(scripts.Script): strength = getattr(p, 'pulid_strength', strength) zero = getattr(p, 'pulid_zero', zero) ortho = getattr(p, 'pulid_ortho', ortho) + sampler = getattr(p, 'pulid_sampler', sampler) + sampler_fn = getattr(self.pulid.sampling, f'sample_{sampler}', None) if shared.sd_model_type == 'sdxl' and not hasattr(shared.sd_model, 'pipe'): try: stdout = io.StringIO() - ctx = contextlib.nullcontext if debug else contextlib.redirect_stdout(stdout) + ctx = contextlib.nullcontext() if debug else contextlib.redirect_stdout(stdout) with ctx: shared.sd_model = self.pulid.StableDiffusionXLPuLIDPipeline( pipe =shared.sd_model, device=devices.device, - sampler=sampler, cache_dir=shared.opts.hfcache_dir, ) shared.sd_model.no_recurse = True @@ -146,6 +145,7 @@ class Script(scripts.Script): errors.display(e, 'PuLID') return None + shared.sd_model.sampler = sampler_fn shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} strength={strength} zero={zero} ortho={ortho} sampler={sampler} images={[i.shape for i in images]}') self.pulid.attention.NUM_ZERO = zero self.pulid.attention.ORTHO = ortho == 'v1' From 3046417584b785b29e3606bd751dd2a8ead69374 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 6 Nov 2024 21:27:41 -0500 Subject: [PATCH 12/40] package logging Signed-off-by: Vladimir Mandic --- modules/loader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/loader.py b/modules/loader.py index 0711c2906..cd51cc8eb 100644 --- a/modules/loader.py +++ b/modules/loader.py @@ -126,4 +126,5 @@ except ImportError: except ImportError: pass # shrug... -errors.log.info(f'System packages: {get_packages()}') +errors.log.info(f'Torch: torch=={torch.__version__} torchvision=={torchvision.__version__}') +errors.log.info(f'Packages: diffusers=={diffusers.__version__} transformers=={transformers.__version__} accelerate=={accelerate.__version__} gradio=={gradio.__version__}') From 13cb5704b99551d8cbea8c1fece4a242e1f0e966 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 6 Nov 2024 22:14:42 -0500 Subject: [PATCH 13/40] pulid img2img and inpaint placeholders Signed-off-by: Vladimir Mandic --- modules/processing_args.py | 4 ++-- modules/pulid/__init__.py | 2 +- modules/pulid/pulid_sdxl.py | 19 +++++++++++++++++++ modules/sd_models.py | 8 +++++++- scripts/pulid_ext.py | 12 ++++++++---- 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/modules/processing_args.py b/modules/processing_args.py index 4cd12a04d..1cb9457c5 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -27,7 +27,7 @@ def task_specific_kwargs(p, model): 'height': 8 * math.ceil(p.height / 8), } elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or is_img2img_model) and len(getattr(p, 'init_images', [])) > 0: - if shared.sd_model_type == 'sdxl': + if shared.sd_model_type == 'sdxl' and hasattr(model, 'register_to_config'): model.register_to_config(requires_aesthetics_score = False) p.ops.append('img2img') task_args = { @@ -55,7 +55,7 @@ def task_specific_kwargs(p, model): 'strength': p.denoising_strength, } elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INPAINTING or is_img2img_model) and len(getattr(p, 'init_images', [])) > 0: - if shared.sd_model_type == 'sdxl': + if shared.sd_model_type == 'sdxl' and hasattr(model, 'register_to_config'): model.register_to_config(requires_aesthetics_score = False) if p.detailer: p.ops.append('detailer') diff --git a/modules/pulid/__init__.py b/modules/pulid/__init__.py index 785b849c2..dcee2d7b9 100644 --- a/modules/pulid/__init__.py +++ b/modules/pulid/__init__.py @@ -5,7 +5,7 @@ Credit and original implementation: import os import sys sys.path.append(os.path.dirname(__file__)) -from pulid_sdxl import StableDiffusionXLPuLIDPipeline +from pulid_sdxl import StableDiffusionXLPuLIDPipeline, StableDiffusionXLPuLIDPipelineImage, StableDiffusionXLPuLIDPipelineInpaint from pulid_utils import resize_numpy_image_long as resize import attention_processor as attention import pulid_sampling as sampling diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index de650b839..af7b8e443 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -307,6 +307,7 @@ class StableDiffusionXLPuLIDPipeline: num_inference_steps: int=50, seed: int=-1, image: np.ndarray=None, + mask_image: np.ndarray=None, strength: float=0.3, id_embedding=None, uncond_id_embedding=None, @@ -356,4 +357,22 @@ class StableDiffusionXLPuLIDPipeline: images = self.pipe.vae.decode(latents).sample images = self.pipe.image_processor.postprocess(images, output_type='pil') + if mask_image is not None: + # TODO: pulid inpaint + # easiest inpaint is to use normal img2img and then combine output with input using mask + # note that mask can be binary or grayscale (soft mask) + raise NotImplementedError('pulid: inpaint') + return images + + +class StableDiffusionXLPuLIDPipelineImage(StableDiffusionXLPuLIDPipeline): + def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, sampler=None, cache_dir=None): # pylint: disable=useless-parent-delegation + super().__init__(pipe, device, sampler, cache_dir) + # we dont do anything special here, just having different class so task-type can be detected/assigned + + +class StableDiffusionXLPuLIDPipelineInpaint(StableDiffusionXLPuLIDPipeline): + def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, sampler=None, cache_dir=None): # pylint: disable=useless-parent-delegation + super().__init__(pipe, device, sampler, cache_dir) + # we dont do anything special here, just having different class so task-type can be detected/assigned diff --git a/modules/sd_models.py b/modules/sd_models.py index 9dd87cec1..5ae14a64a 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1062,7 +1062,6 @@ def set_diffuser_pipe(pipe, new_pipe_type): 'AnimateDiffSDXLPipeline', 'OmniGenPipeline', 'StableDiffusion3ControlNetPipeline', - 'StableDiffusionXLPuLIDPipeline', 'InstantIRPipeline', ] @@ -1084,6 +1083,13 @@ def set_diffuser_pipe(pipe, new_pipe_type): pipe = switch_pipe(diffusers.StableDiffusionPipeline, pipe) if n == 'StableDiffusionXLPAGPipeline': pipe = switch_pipe(diffusers.StableDiffusionXLPipeline, pipe) + if n == 'StableDiffusionXLPuLIDPipeline': + from modules import pulid + if new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: + pipe.__class__ = pulid.StableDiffusionXLPuLIDPipelineImage + else: + pipe.__class__ = pulid.StableDiffusionXLPuLIDPipelineInpaint + return pipe sd_checkpoint_info = getattr(pipe, "sd_checkpoint_info", None) sd_model_checkpoint = getattr(pipe, "sd_model_checkpoint", None) diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 54189cc05..3a157d0fe 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -106,9 +106,10 @@ class Script(scripts.Script): try: from modules import pulid # pylint: disable=redefined-outer-name self.pulid = pulid - # from diffusers import pipelines - # pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["pilid"] = pulid.StableDiffusionXLPuLIDPipeline - # pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["omnigen"] = pulid.StableDiffusionXLPuLIDPipelineImg2Img + from diffusers import pipelines + pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["pulid"] = pulid.StableDiffusionXLPuLIDPipeline + pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["pulid"] = pulid.StableDiffusionXLPuLIDPipelineImage + pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["pulid"] = pulid.StableDiffusionXLPuLIDPipelineInpaint except Exception as e: shared.log.error(f'PuLID: failed to import library: {e}') return None @@ -124,6 +125,8 @@ class Script(scripts.Script): ortho = getattr(p, 'pulid_ortho', ortho) sampler = getattr(p, 'pulid_sampler', sampler) sampler_fn = getattr(self.pulid.sampling, f'sample_{sampler}', None) + if sampler_fn is None: + sampler_fn = self.pulid.sampling.sample_dpmpp_2m_sde if shared.sd_model_type == 'sdxl' and not hasattr(shared.sd_model, 'pipe'): try: @@ -146,7 +149,7 @@ class Script(scripts.Script): return None shared.sd_model.sampler = sampler_fn - shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} strength={strength} zero={zero} ortho={ortho} sampler={sampler} images={[i.shape for i in images]}') + shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} strength={strength} zero={zero} ortho={ortho} sampler={sampler_fn} images={[i.shape for i in images]}') self.pulid.attention.NUM_ZERO = zero self.pulid.attention.ORTHO = ortho == 'v1' self.pulid.attention.ORTHO_v2 = ortho == 'v2' @@ -184,6 +187,7 @@ class Script(scripts.Script): p.task_args['image'] = p.init_images[0] p.task_args['strength'] = p.denoising_strength p.extra_generation_params["PuLID"] = f'Strength={strength} Zero={zero} Ortho={ortho}' + p.extra_generation_params["Sampler"] = sampler if getattr(p, 'xyz', False): # xyz will run its own processing return None processed: processing.Processed = processing.process_images(p) # runs processing using main loop From ce49460b191fc6f817b10ad3ef38cd7e72906156 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 6 Nov 2024 22:29:52 -0500 Subject: [PATCH 14/40] pulid optional keep model loaded Signed-off-by: Vladimir Mandic --- scripts/pulid_ext.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 3a157d0fe..da195b5e4 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -74,14 +74,16 @@ class Script(scripts.Script): with gr.Row(): sampler = gr.Dropdown(label="Sampler", value='dpmpp_sde', choices=['dpmpp_2m', 'dpmpp_2m_sde', 'dpmpp_2s_ancestral', 'dpmpp_3m_sde', 'dpmpp_sde', 'euler', 'euler_ancestral']) ortho = gr.Dropdown(label="Ortho", choices=['off', 'v1', 'v2'], value='v2') + with gr.Row(): + cache = gr.Checkbox(label='Keep model', value=False) with gr.Row(): files = gr.File(label='Input images', file_count='multiple', file_types=['image'], type='file', interactive=True, height=100) with gr.Row(): gallery = gr.Gallery(show_label=False, value=[], visible=False, container=False, rows=1) files.change(fn=self.load_images, inputs=[files], outputs=[gallery]) - return [strength, zero, sampler, ortho, gallery] + return [strength, zero, sampler, ortho, gallery, cache] - def run(self, p: processing.StableDiffusionProcessing, strength: float = 0.8, zero: int = 20, sampler: str = 'dpmpp_sde', ortho: str = 'v2', gallery: list = []): # pylint: disable=arguments-differ + def run(self, p: processing.StableDiffusionProcessing, strength: float = 0.8, zero: int = 20, sampler: str = 'dpmpp_sde', ortho: str = 'v2', gallery: list = [], cache: bool = False): # pylint: disable=arguments-differ, unused-argument images = [] try: if len(gallery) == 0: @@ -197,6 +199,11 @@ class Script(scripts.Script): return processed def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=unused-argument + _strength, _zero, _sampler, _ortho, _gallery, cache = args + cache = getattr(p, 'pulid_cache', cache) + if cache: + shared.log.debug(f'PuLID cache: class={shared.sd_model.__class__.__name__}') + return processed if hasattr(shared.sd_model, 'pipe') and shared.sd_model_type == "sdxl": if hasattr(shared.sd_model, 'app'): shared.sd_model.app = None @@ -204,7 +211,7 @@ class Script(scripts.Script): shared.sd_model.face_helper = None shared.sd_model.clip_vision_model = None shared.sd_model.handler_ante = None - devices.torch_gc(force=True) shared.sd_model = shared.sd_model.pipe - # shared.log.debug(f'PuLID restore: class={shared.sd_model.__class__.__name__}') + devices.torch_gc(force=True) + shared.log.debug(f'PuLID restore: class={shared.sd_model.__class__.__name__}') return processed From 0160b703f30b31222c66c632d17a8fe2896ed165 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 6 Nov 2024 22:40:18 -0500 Subject: [PATCH 15/40] fix xyz duplicate classes Signed-off-by: Vladimir Mandic --- modules/ui_extra_networks.py | 3 ++- scripts/apg.py | 11 ++++++++--- scripts/pulid_ext.py | 12 +++++++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 323f4830f..f6e6cee97 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -148,7 +148,8 @@ class ExtraNetworksPage: if self.title == 'Model': return opt = xyz_grid.AxisOption(f"[Network] {self.title}", str, add_prompt, choices=lambda: [x["name"] for x in self.items]) - xyz_grid.axis_options.append(opt) + if opt not in xyz_grid.axis_options: + xyz_grid.axis_options.append(opt) def link_preview(self, filename): quoted_filename = urllib.parse.quote(filename.replace('\\', '/')) diff --git a/scripts/apg.py b/scripts/apg.py index c7e60c982..6a3020c38 100644 --- a/scripts/apg.py +++ b/scripts/apg.py @@ -32,9 +32,14 @@ class Script(scripts.Script): import sys xyz_classes = [v for k, v in sys.modules.items() if 'xyz_grid_classes' in k][0] - xyz_classes.axis_options.append(xyz_classes.AxisOption("[APG] ETA", float, apply_field("apg_eta"))) - xyz_classes.axis_options.append(xyz_classes.AxisOption("[APG] Momentum", float, apply_field("apg_momentum"))) - xyz_classes.axis_options.append(xyz_classes.AxisOption("[APG] Threshold", float, apply_field("apg_threshold"))) + options = [ + xyz_classes.AxisOption("[APG] ETA", float, apply_field("apg_eta")), + xyz_classes.AxisOption("[APG] Momentum", float, apply_field("apg_momentum")), + xyz_classes.AxisOption("[APG] Threshold", float, apply_field("apg_threshold")), + ] + for option in options: + if option not in xyz_classes.axis_options: + xyz_classes.axis_options.append(option) def run(self, p: processing.StableDiffusionProcessing, eta = 0.0, momentum = 0.0, threshold = 0.0): # pylint: disable=arguments-differ supported_model_list = ['sd', 'sdxl', 'sc'] diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index da195b5e4..05e83ce1f 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -40,9 +40,15 @@ class Script(scripts.Script): import sys xyz_classes = [v for k, v in sys.modules.items() if 'xyz_grid_classes' in k][0] - xyz_classes.axis_options.append(xyz_classes.AxisOption("[PuLID] Strength", float, apply_field("pulid_strength"))) - xyz_classes.axis_options.append(xyz_classes.AxisOption("[PuLID] Zero", int, apply_field("pulid_zero"))) - xyz_classes.axis_options.append(xyz_classes.AxisOption("[PuLID] Ortho", str, apply_field("pulid_ortho"), choices=lambda: ['off', 'v1', 'v2'])) + options = [ + xyz_classes.AxisOption("[PuLID] Strength", float, apply_field("pulid_strength")), + xyz_classes.AxisOption("[PuLID] Zero", int, apply_field("pulid_zero")), + xyz_classes.AxisOption("[PuLID] Ortho", str, apply_field("pulid_ortho"), choices=lambda: ['off', 'v1', 'v2']), + ] + for option in options: + if option not in xyz_classes.axis_options: + xyz_classes.axis_options.append(option) + def load_images(self, files): self.images = [] From 49f1ca2880677e2eb2b705611b7cd90020dbcefe Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 7 Nov 2024 08:05:18 -0500 Subject: [PATCH 16/40] cleanup Signed-off-by: Vladimir Mandic --- modules/pulid/pulid_sdxl.py | 2 -- scripts/pulid_ext.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index af7b8e443..6364309b9 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -155,7 +155,6 @@ class StableDiffusionXLPuLIDPipeline: state_dict_dict[module][new_k] = v for module in state_dict_dict: - print(f'loading from {module}') getattr(self, module).load_state_dict(state_dict_dict[module], strict=True) def to_gray(self, img): @@ -200,7 +199,6 @@ class StableDiffusionXLPuLIDPipeline: align_face = self.face_helper.cropped_faces[0] # incase insightface didn't detect face if id_ante_embedding is None: - print('fail to detect face using insightface, extract embedding on align face') id_ante_embedding = self.handler_ante.get_feat(align_face) id_ante_embedding = torch.from_numpy(id_ante_embedding).to(self.device) diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 05e83ce1f..22aff993f 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -191,9 +191,6 @@ class Script(scripts.Script): p.task_args['id_embedding'] = id_embedding p.task_args['uncond_id_embedding'] = uncond_id_embedding p.task_args['id_scale'] = strength - if len(getattr(p, 'init_images', [])) > 0: - p.task_args['image'] = p.init_images[0] - p.task_args['strength'] = p.denoising_strength p.extra_generation_params["PuLID"] = f'Strength={strength} Zero={zero} Ortho={ortho}' p.extra_generation_params["Sampler"] = sampler if getattr(p, 'xyz', False): # xyz will run its own processing From 94922281751eb094c9b54399a965789885f983c3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 7 Nov 2024 10:08:41 -0500 Subject: [PATCH 17/40] improve auto-pipeline switch Signed-off-by: Vladimir Mandic --- modules/images_namegen.py | 8 ++-- modules/processing_info.py | 4 ++ modules/pulid/pulid_sdxl.py | 16 ++----- modules/sd_models.py | 90 ++++++++++++++++++++----------------- modules/sd_vae.py | 2 +- modules/styles.py | 2 + scripts/pulid_ext.py | 5 ++- 7 files changed, 68 insertions(+), 59 deletions(-) diff --git a/modules/images_namegen.py b/modules/images_namegen.py index d88f85a77..bc58f728a 100644 --- a/modules/images_namegen.py +++ b/modules/images_namegen.py @@ -34,10 +34,10 @@ class FilenameGenerator: 'timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp), 'job_timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp), - 'model': lambda self: shared.sd_model.sd_checkpoint_info.title if shared.sd_loaded else '', - 'model_shortname': lambda self: shared.sd_model.sd_checkpoint_info.model_name if shared.sd_loaded else '', - 'model_name': lambda self: shared.sd_model.sd_checkpoint_info.model_name if shared.sd_loaded else '', - 'model_hash': lambda self: shared.sd_model.sd_checkpoint_info.shorthash if shared.sd_loaded else '', + 'model': lambda self: shared.sd_model.sd_checkpoint_info.title if shared.sd_loaded and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None else '', + 'model_shortname': lambda self: shared.sd_model.sd_checkpoint_info.model_name if shared.sd_loaded and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None else '', + 'model_name': lambda self: shared.sd_model.sd_checkpoint_info.model_name if shared.sd_loaded and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None else '', + 'model_hash': lambda self: shared.sd_model.sd_checkpoint_info.shorthash if shared.sd_loaded and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None else '', 'prompt': lambda self: self.prompt_full(), 'prompt_no_styles': lambda self: self.prompt_no_style(), diff --git a/modules/processing_info.py b/modules/processing_info.py index f721b3717..714ebf35f 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -64,6 +64,10 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No "Comment": comment, "Operations": '; '.join(ops).replace('"', '') if len(p.ops) > 0 else 'none', } + if shared.opts.add_model_name_to_info and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None: + args["Model"] = shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', '') + if shared.opts.add_model_hash_to_info and getattr(shared.sd_model, 'sd_model_hash', None) is not None: + args["Model hash"] = shared.sd_model.sd_model_hash # native if grid is None and (p.n_iter > 1 or p.batch_size > 1) and index >= 0: args['Index'] = f'{p.iteration + 1}x{index + 1}' diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index 6364309b9..0ae603a26 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -91,14 +91,6 @@ class StableDiffusionXLPuLIDPipeline: if sampler is not None: self.sampler = sampler - """ - if sampler == 'dpmpp_sde': - self.sampler = sample_dpmpp_sde - elif sampler == 'dpmpp_2m': - self.sampler = sample_dpmpp_2m - else: - raise NotImplementedError(f'sampler {sampler} not implemented') - """ @property def sigma_min(self): @@ -252,8 +244,8 @@ class StableDiffusionXLPuLIDPipeline: def set_progress_bar_config(self, bar_format: str = None, ncols: int = 80, colour: str = None): import functools from tqdm.auto import trange as trange_orig - import pulid_utils - pulid_utils.trange = functools.partial(trange_orig, bar_format=bar_format, ncols=ncols, colour=colour) + import pulid_sampling + pulid_sampling.trange = functools.partial(trange_orig, bar_format=bar_format, ncols=ncols, colour=colour) def sample(self, x, sigma, **extra_args): x_ddim_space = x / (sigma[:, None, None, None] ** 2 + self.sigma_data**2) ** 0.5 @@ -288,7 +280,7 @@ class StableDiffusionXLPuLIDPipeline: add_noise, ) """ - raise NotImplementedError('pulid: img2img') + raise NotImplementedError(f'PuLID: task=img2img class={self.__class__.__name__} pipe={self.pipe.__class__.__name__} image={image} strength={strength}') else: # standard txt2img will full noise latents = torch.randn((size[0], 4, size[1] // 8, size[2] // 8), device="cpu", generator=torch.manual_seed(seed)) @@ -359,7 +351,7 @@ class StableDiffusionXLPuLIDPipeline: # TODO: pulid inpaint # easiest inpaint is to use normal img2img and then combine output with input using mask # note that mask can be binary or grayscale (soft mask) - raise NotImplementedError('pulid: inpaint') + raise NotImplementedError(f'PuLID: task=inpaint class={self.__class__.__name__} pipe={self.pipe.__class__.__name__} mask_image={mask_image}') return images diff --git a/modules/sd_models.py b/modules/sd_models.py index 5ae14a64a..34ffb7feb 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -780,11 +780,11 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if shared.opts.data.get('sd_model_checkpoint', '') == 'model.safetensors' or shared.opts.data.get('sd_model_checkpoint', '') == '': shared.opts.data['sd_model_checkpoint'] = "stabilityai/stable-diffusion-xl-base-1.0" - if op == 'model' or op == 'dict': - if (model_data.sd_model is not None) and (checkpoint_info is not None) and (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model + if (op == 'model' or op == 'dict'): + if (model_data.sd_model is not None) and (checkpoint_info is not None) and (getattr(model_data.sd_model, 'sd_checkpoint_info', None) is not None) and (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model return else: - if (model_data.sd_refiner is not None) and (checkpoint_info is not None) and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model + if (model_data.sd_refiner is not None) and (checkpoint_info is not None) and (getattr(model_data.sd_refiner, 'sd_checkpoint_info', None) is not None) and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model return sd_model = None @@ -887,7 +887,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No set_diffuser_offload(sd_model, op) if op == 'model' and not (os.path.isdir(checkpoint_info.path) or checkpoint_info.type == 'huggingface'): - sd_vae.apply_vae_config(shared.sd_model.sd_checkpoint_info.filename, vae_file, sd_model) + if getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None: + sd_vae.apply_vae_config(shared.sd_model.sd_checkpoint_info.filename, vae_file, sd_model) if op == 'refiner' and shared.opts.diffusers_move_refiner: shared.log.debug('Moving refiner model to CPU') move_model(sd_model, devices.cpu) @@ -1078,18 +1079,13 @@ def set_diffuser_pipe(pipe, new_pipe_type): if 'Onnx' in pipe.__class__.__name__: return pipe - if new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE or new_pipe_type == DiffusersTaskType.INPAINTING: # in some cases we want to reset the pipeline as they dont have their own variants + new_pipe = None + # in some cases we want to reset the pipeline to parent as they dont have their own variants + if new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE or new_pipe_type == DiffusersTaskType.INPAINTING: if n == 'StableDiffusionPAGPipeline': - pipe = switch_pipe(diffusers.StableDiffusionPipeline, pipe) + new_pipe = switch_pipe(diffusers.StableDiffusionPipeline, pipe) if n == 'StableDiffusionXLPAGPipeline': - pipe = switch_pipe(diffusers.StableDiffusionXLPipeline, pipe) - if n == 'StableDiffusionXLPuLIDPipeline': - from modules import pulid - if new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: - pipe.__class__ = pulid.StableDiffusionXLPuLIDPipelineImage - else: - pipe.__class__ = pulid.StableDiffusionXLPuLIDPipelineInpaint - return pipe + new_pipe = switch_pipe(diffusers.StableDiffusionXLPipeline, pipe) sd_checkpoint_info = getattr(pipe, "sd_checkpoint_info", None) sd_model_checkpoint = getattr(pipe, "sd_model_checkpoint", None) @@ -1101,19 +1097,38 @@ def set_diffuser_pipe(pipe, new_pipe_type): image_encoder = getattr(pipe, "image_encoder", None) feature_extractor = getattr(pipe, "feature_extractor", None) - try: - if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: - new_pipe = diffusers.AutoPipelineForText2Image.from_pipe(pipe) - elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: - new_pipe = diffusers.AutoPipelineForImage2Image.from_pipe(pipe) - elif new_pipe_type == DiffusersTaskType.INPAINTING: - new_pipe = diffusers.AutoPipelineForInpainting.from_pipe(pipe) + if new_pipe is None: + if hasattr(pipe, 'config'): # real pipeline which can be auto-switched + try: + if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: + new_pipe = diffusers.AutoPipelineForText2Image.from_pipe(pipe) + elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: + new_pipe = diffusers.AutoPipelineForImage2Image.from_pipe(pipe) + elif new_pipe_type == DiffusersTaskType.INPAINTING: + new_pipe = diffusers.AutoPipelineForInpainting.from_pipe(pipe) + else: + shared.log.error(f'Pipeline class change failed: type={new_pipe_type} pipeline={pipe.__class__.__name__}') + return pipe + except Exception as e: # pylint: disable=unused-variable + shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}') + return pipe else: - shared.log.error(f'Pipeline class change failed: type={new_pipe_type} pipeline={pipe.__class__.__name__}') - return pipe - except Exception as e: # pylint: disable=unused-variable - shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}') - return pipe + try: # maybe a wrapper pipeline so just change the class + if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: + pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING, pipe.__class__.__name__) # pylint: disable=protected-access + new_pipe = pipe + elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: + pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, pipe.__class__.__name__) # pylint: disable=protected-access + new_pipe = pipe + elif new_pipe_type == DiffusersTaskType.INPAINTING: + pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING, pipe.__class__.__name__) # pylint: disable=protected-access + new_pipe = pipe + else: + shared.log.error(f'Pipeline class change failed: type={new_pipe_type} pipeline={pipe.__class__.__name__}') + return pipe + except Exception as e: # pylint: disable=unused-variable + shared.log.warning(f'Pipeline class set failed: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}') + return pipe # if pipe.__class__ == new_pipe.__class__: # return pipe @@ -1129,8 +1144,12 @@ def set_diffuser_pipe(pipe, new_pipe_type): new_pipe.is_sdxl = getattr(pipe, 'is_sdxl', False) # a1111 compatibility item new_pipe.is_sd2 = getattr(pipe, 'is_sd2', False) new_pipe.is_sd1 = getattr(pipe, 'is_sd1', True) - if hasattr(new_pipe, "watermark"): + if hasattr(new_pipe, 'watermark'): new_pipe.watermark = NoWatermark() + + if hasattr(new_pipe, 'pipe'): # also handle nested pipelines + new_pipe.pipe = set_diffuser_pipe(new_pipe.pipe, new_pipe_type) + fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access shared.log.debug(f"Pipeline class change: original={pipe.__class__.__name__} target={new_pipe.__class__.__name__} device={pipe.device} fn={fn}") # pylint: disable=protected-access pipe = new_pipe @@ -1201,10 +1220,10 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, if checkpoint_info is None: return if op == 'model' or op == 'dict': - if model_data.sd_model is not None and (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model + if (model_data.sd_model is not None) and (getattr(model_data.sd_model, 'sd_checkpoint_info', None) is not None) and (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model return else: - if model_data.sd_refiner is not None and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model + if (model_data.sd_refiner is not None) and (getattr(model_data.sd_refiner, 'sd_checkpoint_info', None) is not None) and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model return shared.log.debug(f'Load {op}: name={checkpoint_info.filename} dict={already_loaded_state_dict is not None}') if timer is None: @@ -1213,12 +1232,12 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, if op == 'model' or op == 'dict': if model_data.sd_model is not None: sd_hijack.model_hijack.undo_hijack(model_data.sd_model) - current_checkpoint_info = model_data.sd_model.sd_checkpoint_info + current_checkpoint_info = getattr(model_data.sd_model, 'sd_checkpoint_info', None) unload_model_weights(op=op) else: if model_data.sd_refiner is not None: sd_hijack.model_hijack.undo_hijack(model_data.sd_refiner) - current_checkpoint_info = model_data.sd_refiner.sd_checkpoint_info + current_checkpoint_info = getattr(model_data.sd_refiner, 'sd_checkpoint_info', None) unload_model_weights(op=op) if not shared.native: @@ -1247,15 +1266,6 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, sd_model = instantiate_from_config(sd_config.model) else: with contextlib.redirect_stdout(stdout): - """ - try: - clip_is_included_into_sd = sd1_clip_weight in state_dict or sd2_clip_weight in state_dict - with sd_disable_initialization.DisableInitialization(disable_clip=clip_is_included_into_sd): - sd_model = instantiate_from_config(sd_config.model) - except Exception as e: - shared.log.error(f'LDM: instantiate from config: {e}') - sd_model = instantiate_from_config(sd_config.model) - """ sd_model = instantiate_from_config(sd_config.model) for line in stdout.getvalue().splitlines(): if len(line) > 0: diff --git a/modules/sd_vae.py b/modules/sd_vae.py index f266f8c38..95ac05c93 100644 --- a/modules/sd_vae.py +++ b/modules/sd_vae.py @@ -289,7 +289,7 @@ def reload_vae_weights(sd_model=None, vae_file=unspecified): if vae_file is not None: shared.log.info(f"VAE weights loaded: {vae_file}") else: - if hasattr(sd_model, "vae") and hasattr(sd_model, "sd_checkpoint_info"): + if hasattr(sd_model, "vae") and getattr(sd_model, "sd_checkpoint_info", None) is not None: vae = load_vae_diffusers(sd_model.sd_checkpoint_info.filename, vae_file, vae_source) if vae is not None: if not hasattr(sd_model, 'original_vae'): diff --git a/modules/styles.py b/modules/styles.py index de9ef43c4..0599bcd86 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -112,6 +112,8 @@ def apply_wildcards_to_prompt(prompt, all_wildcards, seed=-1, silent=False): def get_reference_style(): + if getattr(shared.sd_model, 'sd_checkpoint_info', None) is None: + return None name = shared.sd_model.sd_checkpoint_info.name name = name.replace('\\', '/').replace('Diffusers/', '') for k, v in shared.reference_models.items(): diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 22aff993f..5d209c211 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -8,6 +8,7 @@ from modules import shared, devices, errors, scripts, processing, processing_hel debug = os.environ.get('SD_PULID_DEBUG', None) is not None +direct = False class Script(scripts.Script): @@ -148,7 +149,7 @@ class Script(scripts.Script): ) shared.sd_model.no_recurse = True sd_models.copy_diffuser_options(shared.sd_model, shared.sd_model.pipe) - sd_models.move_model(shared.sd_model, devices.device) # move pipeline to device + # sd_models.move_model(shared.sd_model, devices.device) # move pipeline to device sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model') devices.torch_gc() except Exception as e: @@ -165,7 +166,7 @@ class Script(scripts.Script): shared.sd_model.debug_img_list = [] uncond_id_embedding, id_embedding = shared.sd_model.get_id_embedding(images) - if debug: # run pipeline directly + if direct: # run pipeline directly shared.state.begin('PuLID') processing.fix_seed(p) p.seed = processing_helpers.get_fixed_seed(p.seed) From 6fa55332b8ed48c44c6a821708fd9dc687154318 Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:11:02 -0600 Subject: [PATCH 18/40] pulid img2img, no inpaint yet --- modules/pulid/pulid_sdxl.py | 44 ++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index 0ae603a26..2392219c0 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -261,31 +261,25 @@ class StableDiffusionXLPuLIDPipeline: return latent def init_latent(self, seed, size, image, strength): # pylint: disable=unused-argument + # standard txt2img will full noise + noise = torch.randn((size[0], 4, size[1] // 8, size[2] // 8), device="cpu", generator=torch.manual_seed(seed)) + noise = noise.to(dtype=self.pipe.unet.dtype, device=self.device) if image is not None and strength > 0: - # TODO pulid img2img - # input can be PIL.Image or np.ndarray so it needs to be converted to rgb tensor - # image must be resized, encoded and noised according to denoising strength - # see below for example from StableDiffusionXLImg2ImgPipeline - latents = None - """ - image = self.image_processor.preprocess(image) - latents = self.prepare_latents( + image = self.pipe.image_processor.preprocess(image) + latents = self.pipe.prepare_latents( image, - latent_timestep, - batch_size, - num_images_per_prompt, - prompt_embeds.dtype, - device, - generator, - add_noise, + None, # timestep (not needed) + 1, # batch_size + 1, # num_images_per_prompt + noise.dtype, + noise.device, + None, # generator + False, # add_noise ) - """ - raise NotImplementedError(f'PuLID: task=img2img class={self.__class__.__name__} pipe={self.pipe.__class__.__name__} image={image} strength={strength}') else: - # standard txt2img will full noise - latents = torch.randn((size[0], 4, size[1] // 8, size[2] // 8), device="cpu", generator=torch.manual_seed(seed)) - latents = latents.to(dtype=self.pipe.unet.dtype, device=self.device) - return latents + latents = torch.zeros_like(noise) + + return latents, noise def __call__( self, @@ -309,10 +303,14 @@ class StableDiffusionXLPuLIDPipeline: size = (1, height, width) # sigmas sigmas = self.get_sigmas_karras(num_inference_steps).to(self.device) + if image is not None and strength > 0: + _, num_inference_steps = self.pipe.get_timesteps(num_inference_steps, strength, self.device, None) # denoising_start disabled + sigmas = sigmas[-(num_inference_steps + 1):].to(self.device) # shorten sigmas in i2i + # latents - noise = self.init_latent(seed, size, image, strength) - latents = noise * sigmas[0].to(noise) + latents, noise = self.init_latent(seed, size, image, strength) + latents = latents + noise * sigmas[0].to(noise) ( prompt_embeds, From 8249865f41b22481be782af34fcaf808685d0fa3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 7 Nov 2024 12:00:15 -0500 Subject: [PATCH 19/40] fix pag switch pipeline Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 5 +++-- modules/sd_models.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e08cb147..123cfdbf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-11-06 +## Update for 2024-11-07 Smaller release just few days after the last one, but with some important fixes and improvements. This release can be considered an LTS release before we kick off the next round of major updates. @@ -9,7 +9,7 @@ This release can be considered an LTS release before we kick off the next round - add built-in [changelog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) search since changelog is the best up-to-date source of info go to system -> changelog and search/highligh/navigate directly in UI! - + - major [Wiki](https://github.com/vladmandic/automatic/wiki) updates - Integrations: - [PuLID](https://github.com/ToTheBeginning/PuLID): Pure and Lightning ID Customization via Contrastive Alignment - advanced method of face transfer with better quality as well as control over identity and appearance @@ -74,6 +74,7 @@ This release can be considered an LTS release before we kick off the next round - added `cli/model-keys.py` to quicky display content of any safetensors file - Internal: - Repo: move screenshots to GH pages + - Auto pipeline switching coveres wrapper classes and nested pipelines - Fixes: - custom watermark add alphablending diff --git a/modules/sd_models.py b/modules/sd_models.py index 34ffb7feb..1f932bd30 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1083,9 +1083,9 @@ def set_diffuser_pipe(pipe, new_pipe_type): # in some cases we want to reset the pipeline to parent as they dont have their own variants if new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE or new_pipe_type == DiffusersTaskType.INPAINTING: if n == 'StableDiffusionPAGPipeline': - new_pipe = switch_pipe(diffusers.StableDiffusionPipeline, pipe) + pipe = switch_pipe(diffusers.StableDiffusionPipeline, pipe) if n == 'StableDiffusionXLPAGPipeline': - new_pipe = switch_pipe(diffusers.StableDiffusionXLPipeline, pipe) + pipe = switch_pipe(diffusers.StableDiffusionXLPipeline, pipe) sd_checkpoint_info = getattr(pipe, "sd_checkpoint_info", None) sd_model_checkpoint = getattr(pipe, "sd_model_checkpoint", None) From 77db0c9768060d11c1283e62b4c2c712f7446796 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 7 Nov 2024 12:06:35 -0500 Subject: [PATCH 20/40] update changelog Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 123cfdbf9..cdf7163f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ This release can be considered an LTS release before we kick off the next round - advanced method of face transfer with better quality as well as control over identity and appearance try it out, likely the best quality available for sdxl models - select in *scripts -> pulid* - - compatible with *sdxl* + - compatible with *sdxl* for text-to-image and image-to-image - can be used in xyz grid - *note*: this module contains several advanced features on top of original implementation - [InstantIR](https://github.com/instantX-research/InstantIR): Blind Image Restoration with Instant Generative Reference From 9b5c0c738d553cb7390388bf497f5dc04bad0f47 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 7 Nov 2024 12:40:36 -0500 Subject: [PATCH 21/40] update Signed-off-by: Vladimir Mandic --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a2caa5cb7..d39fb7563 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG - Built-in Control for Text, Image, Batch and video processing! â–¹ **ControlNet | ControlNet XS | Control LLLite | T2I Adapters | IP Adapters** - Multiplatform! - â–¹ **Windows | Linux | MacOS with CPU | nVidia | AMD | IntelArc/IPEX | DirectML | OpenVINO | ONNX+Olive | ZLUDA** + â–¹ **Windows | Linux | MacOS | nVidia | AMD | IntelArc/IPEX | DirectML | OpenVINO | ONNX+Olive | ZLUDA** - Platform specific autodetection and tuning performed on install - Optimized processing with latest `torch` developments with built-in support for `torch.compile` and multiple compile backends: *Triton, ZLUDA, StableFast, DeepCache, OpenVINO, NNCF, IPEX, OneDiff* From 9fef22735fd758d92758ab0ebb0e9ddb26cc2b4b Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Fri, 8 Nov 2024 10:04:02 -0600 Subject: [PATCH 22/40] pulid inpaint, XYZ broken --- modules/pulid/pulid_sampling.py | 37 +++++++++++++--- modules/pulid/pulid_sdxl.py | 76 ++++++++++++++++++++++++--------- 2 files changed, 86 insertions(+), 27 deletions(-) diff --git a/modules/pulid/pulid_sampling.py b/modules/pulid/pulid_sampling.py index 9996f035a..e319c0d27 100644 --- a/modules/pulid/pulid_sampling.py +++ b/modules/pulid/pulid_sampling.py @@ -67,6 +67,15 @@ def default_noise_sampler(x): return lambda sigma, sigma_next: torch.randn_like(x) +def inpaint_mask(x, i, steps, mask_args): + noised_original = mask_args["latent"].clone().to(x) + latent_mask = mask_args["latent_mask"].to(x) + if i < steps: + noised_original += mask_args["noise"].to(x) * mask_args["sigmas"][i+1].to(x) + x = (latent_mask * x) + ((1 - latent_mask) * noised_original.to(x)) + return x + + class BatchedBrownianTree: """A wrapper around torchsde.BrownianTree that enables batches of entropy.""" @@ -120,7 +129,7 @@ class BrownianTreeNoiseSampler: @torch.no_grad() -def sample_euler(model, x, sigmas, extra_args=None, callback=None, disable=None, s_churn=0., s_tmin=0., s_tmax=float('inf'), s_noise=1.): +def sample_euler(model, x, sigmas, extra_args=None, callback=None, disable=None, s_churn=0., s_tmin=0., s_tmax=float('inf'), s_noise=1., mask_args=None): """Implements Algorithm 2 (Euler steps) from Karras et al. (2022).""" extra_args = {} if extra_args is None else extra_args s_in = x.new_ones([x.shape[0]]) @@ -137,11 +146,13 @@ def sample_euler(model, x, sigmas, extra_args=None, callback=None, disable=None, dt = sigmas[i + 1] - sigma_hat # Euler method x = x + (d * dt).to(x.dtype) + if mask_args is not None: + x = inpaint_mask(x, i, len(sigmas) - 2, mask_args) return x @torch.no_grad() -def sample_euler_ancestral(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None): +def sample_euler_ancestral(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, mask_args=None): """Ancestral sampling with Euler method steps.""" extra_args = {} if extra_args is None else extra_args noise_sampler = default_noise_sampler(x) if noise_sampler is None else noise_sampler @@ -157,6 +168,8 @@ def sample_euler_ancestral(model, x, sigmas, extra_args=None, callback=None, dis x = x + (d * dt).to(x.dtype) if sigmas[i + 1] > 0: x = x + (noise_sampler(sigmas[i], sigmas[i + 1]) * s_noise * sigma_up).to(x.dtype) + if mask_args is not None: + x = inpaint_mask(x, i, len(sigmas) - 2, mask_args) return x @@ -375,7 +388,7 @@ class DPMSolver(nn.Module): @torch.no_grad() -def sample_dpmpp_2s_ancestral(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None): +def sample_dpmpp_2s_ancestral(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, mask_args=None): """Ancestral sampling with DPM-Solver++(2S) second-order steps.""" extra_args = {} if extra_args is None else extra_args noise_sampler = default_noise_sampler(x) if noise_sampler is None else noise_sampler @@ -405,11 +418,13 @@ def sample_dpmpp_2s_ancestral(model, x, sigmas, extra_args=None, callback=None, # Noise addition if sigmas[i + 1] > 0: x = x + noise_sampler(sigmas[i], sigmas[i + 1]) * s_noise * sigma_up + if mask_args is not None: + x = inpaint_mask(x, i, len(sigmas) - 2, mask_args) return x @torch.no_grad() -def sample_dpmpp_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, r=1 / 2): +def sample_dpmpp_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, r=1 / 2, mask_args=None): """DPM-Solver++ (stochastic).""" sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max() noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max) if noise_sampler is None else noise_sampler @@ -447,11 +462,13 @@ def sample_dpmpp_sde(model, x, sigmas, extra_args=None, callback=None, disable=N denoised_d = (1 - fac) * denoised + fac * denoised_2 x = (sigma_fn(t_next_) / sigma_fn(t)) * x - (t - t_next_).expm1() * denoised_d x = x + noise_sampler(sigma_fn(t), sigma_fn(t_next)) * s_noise * su + if mask_args is not None: + x = inpaint_mask(x, i, len(sigmas) - 2, mask_args) return x @torch.no_grad() -def sample_dpmpp_2m(model, x, sigmas, extra_args=None, callback=None, disable=None): +def sample_dpmpp_2m(model, x, sigmas, extra_args=None, callback=None, disable=None, mask_args=None): """DPM-Solver++(2M).""" extra_args = {} if extra_args is None else extra_args s_in = x.new_ones([x.shape[0]]) @@ -473,11 +490,13 @@ def sample_dpmpp_2m(model, x, sigmas, extra_args=None, callback=None, disable=No denoised_d = (1 + 1 / (2 * r)) * denoised - (1 / (2 * r)) * old_denoised x = (sigma_fn(t_next) / sigma_fn(t)) * x - (-h).expm1() * denoised_d old_denoised = denoised + if mask_args is not None: + x = inpaint_mask(x, i, len(sigmas) - 2, mask_args) return x @torch.no_grad() -def sample_dpmpp_2m_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, solver_type='midpoint'): +def sample_dpmpp_2m_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, solver_type='midpoint', mask_args=None): """DPM-Solver++(2M) SDE.""" if solver_type not in {'heun', 'midpoint'}: @@ -518,11 +537,13 @@ def sample_dpmpp_2m_sde(model, x, sigmas, extra_args=None, callback=None, disabl old_denoised = denoised h_last = h + if mask_args is not None: + x = inpaint_mask(x, i, len(sigmas) - 2, mask_args) return x @torch.no_grad() -def sample_dpmpp_3m_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None): +def sample_dpmpp_3m_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, mask_args=None): """DPM-Solver++(3M) SDE.""" sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max() @@ -568,4 +589,6 @@ def sample_dpmpp_3m_sde(model, x, sigmas, extra_args=None, callback=None, disabl denoised_1, denoised_2 = denoised, denoised_1 h_1, h_2 = h, h_1 + if mask_args is not None: + x = inpaint_mask(x, i, len(sigmas) - 2, mask_args) return x diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index 2392219c0..fade7509d 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -260,22 +260,40 @@ class StableDiffusionXLPuLIDPipeline: self.callback_on_step_end(self.pipe, step=self.step, timestep=t, kwargs={ 'latents': latent }) return latent - def init_latent(self, seed, size, image, strength): # pylint: disable=unused-argument + def init_latent(self, seed, size, image, mask_image, strength, width, height): # pylint: disable=unused-argument # standard txt2img will full noise noise = torch.randn((size[0], 4, size[1] // 8, size[2] // 8), device="cpu", generator=torch.manual_seed(seed)) noise = noise.to(dtype=self.pipe.unet.dtype, device=self.device) - if image is not None and strength > 0: + if strength > 0 and image is not None: image = self.pipe.image_processor.preprocess(image) - latents = self.pipe.prepare_latents( - image, - None, # timestep (not needed) - 1, # batch_size - 1, # num_images_per_prompt - noise.dtype, - noise.device, - None, # generator - False, # add_noise - ) + if mask_image is not None: # Inpaint + latents = self.pipe.prepare_latents(1, # batch_size, + self.pipe.vae.config.latent_channels, # num_channels_latents + height, + width, + noise.dtype, + noise.device, + None, # generator + latents=None, + image=image, + timestep=1000, + is_strength_max=False, + add_noise=False, + return_noise=False, + return_image_latents=False, + ) + latents = latents[0] + else: # img2img + latents = self.pipe.prepare_latents(image, + None, # timestep (not needed) + 1, # batch_size + 1, # num_images_per_prompt + noise.dtype, + noise.device, + None, # generator + False, # add_noise + ) + else: latents = torch.zeros_like(noise) @@ -309,8 +327,8 @@ class StableDiffusionXLPuLIDPipeline: # latents - latents, noise = self.init_latent(seed, size, image, strength) - latents = latents + noise * sigmas[0].to(noise) + latent, noise = self.init_latent(seed, size, image, mask_image, strength, width, height) + noisy_latent = latent + noise * sigmas[0].to(noise) ( prompt_embeds, @@ -339,17 +357,35 @@ class StableDiffusionXLPuLIDPipeline: cross_attention_kwargs={'id_embedding': uncond_id_embedding, 'id_scale': id_scale}, ), ) + if mask_image is not None: + latent_mask = torch.Tensor(np.asarray(mask_image.convert("L").resize((noisy_latent.shape[-1], noisy_latent.shape[-2])))).reshape((noisy_latent.shape[-2], noisy_latent.shape[-1])) + latent_mask /= latent_mask.max() + mask_args = dict( + latent=latent, + latent_mask=latent_mask, + noise=noise, + sigmas=sigmas, + ) + else: + mask_args = None - latents = self.sampler(self.sample, latents, sigmas, extra_args=sampler_kwargs, disable=False) + latents = self.sampler(self.sample, noisy_latent, sigmas, extra_args=sampler_kwargs, disable=False, mask_args=mask_args) latents = latents.to(dtype=self.pipe.vae.dtype, device=self.device) / self.pipe.vae.config.scaling_factor images = self.pipe.vae.decode(latents).sample images = self.pipe.image_processor.postprocess(images, output_type='pil') - if mask_image is not None: - # TODO: pulid inpaint - # easiest inpaint is to use normal img2img and then combine output with input using mask - # note that mask can be binary or grayscale (soft mask) - raise NotImplementedError(f'PuLID: task=inpaint class={self.__class__.__name__} pipe={self.pipe.__class__.__name__} mask_image={mask_image}') + # Pixel space final mask + # if mask_image is not None: + # # TODO: Fix XYZ + # from PIL import Image + # mask_image = np.asarray(mask_image.convert("L")) + # mask_image = mask_image / mask_image.max() + # mask_image = mask_image.reshape(1,mask_image.shape[0],mask_image.shape[1],1) + # image = np.asarray(image).astype(mask_image.dtype) + # images = np.asarray(images).astype(mask_image.dtype) + # images = ((1 - mask_image) * image) + (mask_image * images) + # images = images[0].round().astype(np.uint8) + # images = [Image.fromarray(images)] return images From 3f2d3d208d9133193abf9b0c46ec1bca0b9577a8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 7 Nov 2024 19:44:53 -0500 Subject: [PATCH 23/40] wiki search Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 4 ++- javascript/changelog.js | 7 +++++ javascript/sdnext.css | 5 ++++ modules/ui.py | 18 ++++------- modules/ui_docs.py | 66 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 modules/ui_docs.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cdf7163f8..881ba67cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,11 @@ Smaller release just few days after the last one, but with some important fixes This release can be considered an LTS release before we kick off the next round of major updates. - Docs: - - add built-in [changelog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) search + - UI built-in [changelog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) search since changelog is the best up-to-date source of info go to system -> changelog and search/highligh/navigate directly in UI! + - UI built-in [wiki](https://github.com/vladmandic/automatic/wiki) + go to system -> wiki and search wiki pages directly in UI! - major [Wiki](https://github.com/vladmandic/automatic/wiki) updates - Integrations: - [PuLID](https://github.com/ToTheBeginning/PuLID): Pure and Lightning ID Customization via Contrastive Alignment diff --git a/javascript/changelog.js b/javascript/changelog.js index 80c9956b8..97dc9daf5 100644 --- a/javascript/changelog.js +++ b/javascript/changelog.js @@ -75,3 +75,10 @@ async function initChangelog() { }; search.addEventListener('keyup', searchChangelog); } + +function wikiSearch(txt) { + log('wikiSearch', txt); + const url = `https://github.com/search?q=repo%3Avladmandic%2Fautomatic+${encodeURIComponent(txt)}&type=wikis`; + // window.open(url, '_blank').focus(); + return txt; +} diff --git a/javascript/sdnext.css b/javascript/sdnext.css index d412d966f..192d0d487 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -326,6 +326,11 @@ div:has(>#tab-gallery-folders) { flex-grow: 0 !important; background-color: var( .changelog_arrow:hover { background-color: var(--button-primary-border-color-hover); } .changelog_highlight { background-color: var(--color-warning); } +/* wiki */ +#wiki_result > div > div { padding: 0.5em; margin-right: 2em; } +#wiki_result li { display: block; } +#wiki_result h3 { background-color: var(--background-fill-primary); margin: 0; padding: 0.3em; margin-bottom: 0.2em; } + /* loader */ .splash { position: fixed; top: 0; left: 0; width: 100vw; height: 100vh; z-index: 1000; display: block; text-align: center; } .motd { margin-top: 2em; color: var(--body-text-color-subdued); font-family: monospace; font-variant: all-petite-caps; } diff --git a/modules/ui.py b/modules/ui.py index 039ef6487..4a9d35728 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -355,20 +355,12 @@ def create_ui(startup_timer = None): ui_onnx.create_ui() with gr.TabItem("Change log", id="change_log", elem_id="system_tab_changelog"): - def get_changelog(): - with open('CHANGELOG.md', 'r', encoding='utf-8') as f: - content = f.read() - content = content.replace('# Change Log for SD.Next', ' ') - return content + from modules import ui_docs + ui_docs.create_ui_logs() - with gr.Column(): - get_changelog_btn = gr.Button(value='Get changelog', elem_id="get_changelog") - with gr.Column(): - _changelog_search = gr.Textbox(label="Search", elem_id="changelog_search") - _changelog_result = gr.HTML(elem_id="changelog_result") - - changelog_markdown = gr.Markdown('', elem_id="changelog_markdown") - get_changelog_btn.click(fn=get_changelog, outputs=[changelog_markdown], show_progress=True) + with gr.TabItem("Wiki", id="wiki", elem_id="system_tab_wiki"): + from modules import ui_docs + ui_docs.create_ui_wiki() def unload_sd_weights(): modules.sd_models.unload_model_weights(op='model') diff --git a/modules/ui_docs.py b/modules/ui_docs.py new file mode 100644 index 000000000..7f85aa851 --- /dev/null +++ b/modules/ui_docs.py @@ -0,0 +1,66 @@ +import gradio as gr +from modules import ui_symbols, ui_components + + +def create_ui_logs(): + def get_changelog(): + with open('CHANGELOG.md', 'r', encoding='utf-8') as f: + content = f.read() + content = content.replace('# Change Log for SD.Next', ' ') + return content + + with gr.Column(): + get_changelog_btn = gr.Button(value='Get changelog', elem_id="get_changelog") + gr.HTML('  Open GitHub Changelog') + with gr.Column(): + _changelog_search = gr.Textbox(label="Search Changelog", elem_id="changelog_search") + _changelog_result = gr.HTML(elem_id="changelog_result") + + changelog_markdown = gr.Markdown('', elem_id="changelog_markdown") + get_changelog_btn.click(fn=get_changelog, outputs=[changelog_markdown], show_progress=True) + + +def create_ui_wiki(): + def search_github(search_term): + import requests + from urllib.parse import quote + from installer import install + + install('beautifulsoup4') + from bs4 import BeautifulSoup + + url = f'https://github.com/search?q=repo%3Avladmandic%2Fautomatic+{quote(search_term)}&type=wikis' + res = requests.get(url, timeout=10) + if res.status_code == 200: + html = res.content + soup = BeautifulSoup(html, 'html.parser') + + # remove header links + tags = soup.find_all(attrs={"data-hovercard-url": "/vladmandic/automatic/hovercard"}) + for tag in tags: + tag.extract() + + # replace relative links with full links + tags = soup.find_all('a') + for tag in tags: + if tag.has_attr('href') and tag['href'].startswith('/'): + tag['href'] = 'https://github.com' + tag['href'] + + # find result only + result = soup.find(attrs={"data-testid": "results-list"}) + if result is None: + return 'No results found' + html = str(result) + return html + else: + return f'Error: {res.status_code}' + + with gr.Row(): + gr.HTML('  Open GitHub Wiki') + with gr.Row(): + wiki_search = gr.Textbox(label="Search Wiki Pages", elem_id="wiki_search") + wiki_search_btn = ui_components.ToolButton(value=ui_symbols.search, label="Search", elem_id="wiki_search_btn") + with gr.Row(): + wiki_result = gr.HTML(elem_id="wiki_result", value='test') + wiki_search.submit(_js="wikiSearch", fn=search_github, inputs=[wiki_search], outputs=[wiki_result]) + wiki_search_btn.click(_js="wikiSearch", fn=search_github, inputs=[wiki_search], outputs=[wiki_result]) From 3077aaf4c0972adab1bb4c402afef427e2322ede Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 7 Nov 2024 20:45:34 -0500 Subject: [PATCH 24/40] add info tab Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 5 +++-- modules/ui.py | 19 +++++++++++-------- modules/ui_docs.py | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 881ba67cf..3626d249f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,12 @@ Smaller release just few days after the last one, but with some important fixes This release can be considered an LTS release before we kick off the next round of major updates. - Docs: + - new top-level **info** tab with access to [changelog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) and [wiki](https://github.com/vladmandic/automatic/wiki) - UI built-in [changelog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) search since changelog is the best up-to-date source of info - go to system -> changelog and search/highligh/navigate directly in UI! + go to info -> changelog and search/highligh/navigate directly in UI! - UI built-in [wiki](https://github.com/vladmandic/automatic/wiki) - go to system -> wiki and search wiki pages directly in UI! + go to info -> wiki and search wiki pages directly in UI! - major [Wiki](https://github.com/vladmandic/automatic/wiki) updates - Integrations: - [PuLID](https://github.com/ToTheBeginning/PuLID): Pure and Lightning ID Customization via Contrastive Alignment diff --git a/modules/ui.py b/modules/ui.py index 4a9d35728..4490bbf8c 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -354,14 +354,6 @@ def create_ui(startup_timer = None): from modules.onnx_impl import ui as ui_onnx ui_onnx.create_ui() - with gr.TabItem("Change log", id="change_log", elem_id="system_tab_changelog"): - from modules import ui_docs - ui_docs.create_ui_logs() - - with gr.TabItem("Wiki", id="wiki", elem_id="system_tab_wiki"): - from modules import ui_docs - ui_docs.create_ui_wiki() - def unload_sd_weights(): modules.sd_models.unload_model_weights(op='model') modules.sd_models.unload_model_weights(op='refiner') @@ -382,6 +374,16 @@ def create_ui(startup_timer = None): timer.startup.record("ui-settings") + with gr.Blocks(analytics_enabled=False) as info_interface: + with gr.Tabs(elem_id="tabs_info"): + with gr.TabItem("Change log", id="change_log", elem_id="system_tab_changelog"): + from modules import ui_docs + ui_docs.create_ui_logs() + + with gr.TabItem("Wiki", id="wiki", elem_id="system_tab_wiki"): + from modules import ui_docs + ui_docs.create_ui_wiki() + interfaces = [] interfaces += [(txt2img_interface, "Text", "txt2img")] interfaces += [(img2img_interface, "Image", "img2img")] @@ -391,6 +393,7 @@ def create_ui(startup_timer = None): interfaces += [(models_interface, "Models", "models")] interfaces += script_callbacks.ui_tabs_callback() interfaces += [(settings_interface, "System", "system")] + interfaces += [(info_interface, "Info", "info")] from modules import ui_extensions extensions_interface = ui_extensions.create_ui() diff --git a/modules/ui_docs.py b/modules/ui_docs.py index 7f85aa851..08159beb3 100644 --- a/modules/ui_docs.py +++ b/modules/ui_docs.py @@ -61,6 +61,6 @@ def create_ui_wiki(): wiki_search = gr.Textbox(label="Search Wiki Pages", elem_id="wiki_search") wiki_search_btn = ui_components.ToolButton(value=ui_symbols.search, label="Search", elem_id="wiki_search_btn") with gr.Row(): - wiki_result = gr.HTML(elem_id="wiki_result", value='test') + wiki_result = gr.HTML(elem_id="wiki_result", value='') wiki_search.submit(_js="wikiSearch", fn=search_github, inputs=[wiki_search], outputs=[wiki_result]) wiki_search_btn.click(_js="wikiSearch", fn=search_github, inputs=[wiki_search], outputs=[wiki_result]) From 0cf3283ac81b85d8493ac54e4821562661ad2d4d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 8 Nov 2024 08:26:11 -0500 Subject: [PATCH 25/40] update docs Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 4 +- README.md | 159 ++++++++++------------------------------------ cli/README.md | 116 --------------------------------- modules/shared.py | 97 ++++++++++++++-------------- 4 files changed, 84 insertions(+), 292 deletions(-) delete mode 100644 cli/README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3626d249f..d78943f42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-11-07 +## Update for 2024-11-08 Smaller release just few days after the last one, but with some important fixes and improvements. This release can be considered an LTS release before we kick off the next round of major updates. @@ -12,7 +12,7 @@ This release can be considered an LTS release before we kick off the next round go to info -> changelog and search/highligh/navigate directly in UI! - UI built-in [wiki](https://github.com/vladmandic/automatic/wiki) go to info -> wiki and search wiki pages directly in UI! - - major [Wiki](https://github.com/vladmandic/automatic/wiki) updates + - major [Wiki](https://github.com/vladmandic/automatic/wiki) and [Home](https://github.com/vladmandic/automatic) updates - Integrations: - [PuLID](https://github.com/ToTheBeginning/PuLID): Pure and Lightning ID Customization via Contrastive Alignment - advanced method of face transfer with better quality as well as control over identity and appearance diff --git a/README.md b/README.md index d39fb7563..d099496b8 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@
-SD.Next +SD.Next -**Stable Diffusion implementation with advanced features** +**Image Diffusion implementation with advanced features** -[![Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/vladmandic) -![Last Commit](https://img.shields.io/github/last-commit/vladmandic/automatic?svg=true) +![Last update](https://img.shields.io/github/last-commit/vladmandic/automatic?svg=true) ![License](https://img.shields.io/github/license/vladmandic/automatic?svg=true) [![Discord](https://img.shields.io/discord/1101998836328697867?logo=Discord&svg=true)](https://discord.gg/VjvR2tabEX) +[![Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/vladmandic) [Wiki](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.gg/VjvR2tabEX) | [Changelog](CHANGELOG.md) @@ -18,45 +18,36 @@ - [SD.Next Features](#sdnext-features) - [Model support](#model-support) - [Platform support](#platform-support) -- [Backend support](#backend-support) -- [Examples](#examples) -- [Install](#install) -- [Notes](#notes) +- [Getting started](#getting-started) ## SD.Next Features All individual features are not listed here, instead check [ChangeLog](CHANGELOG.md) for full list of changes -- Multiple backends! - ▹ **Diffusers | Original** - Multiple UIs! ▹ **Standard | Modern** - Multiple diffusion models! - ▹ **Stable Diffusion 1.5/2.1/XL/3.0/3.5 | LCM | Lightning | Segmind | Kandinsky | Pixart-α | Pixart-Σ | Stable Cascade | FLUX.1 | AuraFlow | Würstchen | Alpha Lumina | Kwai Kolors | aMUSEd | DeepFloyd IF | UniDiffusion | SD-Distilled | BLiP Diffusion | KOALA | SDXS | Hyper-SD | HunyuanDiT | CogView | OmniGen | Meissonic | etc.** - Built-in Control for Text, Image, Batch and video processing! - ▹ **ControlNet | ControlNet XS | Control LLLite | T2I Adapters | IP Adapters** - Multiplatform! ▹ **Windows | Linux | MacOS | nVidia | AMD | IntelArc/IPEX | DirectML | OpenVINO | ONNX+Olive | ZLUDA** -- Platform specific autodetection and tuning performed on install +- Multiple backends! + ▹ **Diffusers | Original** +- Platform specific autodetection and tuning performed on install - Optimized processing with latest `torch` developments with built-in support for `torch.compile` and multiple compile backends: *Triton, ZLUDA, StableFast, DeepCache, OpenVINO, NNCF, IPEX, OneDiff* - Improved prompt parser -- Enhanced *Lora*/*LoCon*/*Lyco* code supporting latest trends in training - Built-in queue management - Enterprise level logging and hardened API - Built in installer with automatic updates and dependency management -- Modernized UI with theme support and number of built-in themes *(dark and light)* -- Mobile compatible +- Mobile compatible
*Main interface using **StandardUI***: -![screenshot-text2image](https://github.com/user-attachments/assets/87ac2813-65c2-45f4-80b8-67b26ccf5cd6) +![screenshot-standardui](https://github.com/user-attachments/assets/cab47fe3-9adb-4d67-aea9-9ee738df5dcc) *Main interface using **ModernUI***: -![screenshot-modernui-f1](https://github.com/user-attachments/assets/b509a280-8d3b-48b5-8525-363bad8c1ed2) -![screenshot-modernui](https://github.com/user-attachments/assets/fef33127-f733-4e78-b66e-17729539512f) -![screenshot-modernui-sd3](https://github.com/user-attachments/assets/1ed02ecc-23e4-4fda-8ae5-2d7393dc530c) +![screenshot-modernui](https://github.com/user-attachments/assets/39e3bc9a-a9f7-4cda-ba33-7da8def08032) For screenshots and informations on other available themes, see [Themes Wiki](https://github.com/vladmandic/automatic/wiki/Themes) @@ -65,12 +56,10 @@ For screenshots and informations on other available themes, see [Themes Wiki](ht ## Model support Additional models will be added as they become available and there is public interest in them -See [models overview](https://github.com/vladmandic/automatic/wiki/Models) for details on each model, including their architecture, complexity and other info +See [models overview](wiki/Models) for details on each model, including their architecture, complexity and other info - [RunwayML Stable Diffusion](https://github.com/Stability-AI/stablediffusion/) 1.x and 2.x *(all variants)* -- [StabilityAI Stable Diffusion XL](https://github.com/Stability-AI/generative-models) -- [StabilityAI Stable Diffusion](https://stability.ai/news/stable-diffusion-3-medium) -- [Stable Diffusion 3.x](https://huggingface.co/stabilityai/stable-diffusion-3.5-large) 3.0 Medium, 3.5 Medium, 3.5 Large, 3.5 Large Turbo +- [StabilityAI Stable Diffusion XL](https://github.com/Stability-AI/generative-models), [StabilityAI Stable Diffusion 3.0](https://stability.ai/news/stable-diffusion-3-medium) Medium, [StabilityAI Stable Diffusion 3.5](https://huggingface.co/stabilityai/stable-diffusion-3.5-large) Medium, Large, Large Turbo - [StabilityAI Stable Video Diffusion](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid) Base, XT 1.0, XT 1.1 - [StabilityAI Stable Cascade](https://github.com/Stability-AI/StableCascade) *Full* and *Lite* - [Black Forest Labs FLUX.1](https://blackforestlabs.ai/announcing-black-forest-labs/) Dev, Schnell @@ -84,13 +73,9 @@ See [models overview](https://github.com/vladmandic/automatic/wiki/Models) for d - [CogView 3+](https://huggingface.co/THUDM/CogView3-Plus-3B) - [LCM: Latent Consistency Models](https://github.com/openai/consistency_models) - [aMUSEd](https://huggingface.co/amused/amused-256) 256 and 512 -- [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega) -- [Segmind SSD-1B](https://huggingface.co/segmind/SSD-1B) -- [Segmind SegMoE](https://github.com/segmind/segmoe) *SD and SD-XL* -- [Segmind SD Distilled](https://huggingface.co/blog/sd_distillation) *(all variants)* +- [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega), [Segmind SSD-1B](https://huggingface.co/segmind/SSD-1B), [Segmind SegMoE](https://github.com/segmind/segmoe) *SD and SD-XL*, [Segmind SD Distilled](https://huggingface.co/blog/sd_distillation) *(all variants)* - [Kandinsky](https://github.com/ai-forever/Kandinsky-2) *2.1 and 2.2 and latest 3.0* -- [PixArt-α XL 2](https://github.com/PixArt-alpha/PixArt-alpha) *Medium and Large* -- [PixArt-Σ](https://github.com/PixArt-alpha/PixArt-sigma) +- [PixArt-α XL 2](https://github.com/PixArt-alpha/PixArt-alpha) *Medium and Large*, [PixArt-Σ](https://github.com/PixArt-alpha/PixArt-sigma) - [Warp Wuerstchen](https://huggingface.co/blog/wuertschen) - [Tsinghua UniDiffusion](https://github.com/thu-ml/unidiffuser) - [DeepFloyd IF](https://github.com/deep-floyd/IF) *Medium and Large* @@ -101,15 +86,6 @@ See [models overview](https://github.com/vladmandic/automatic/wiki/Models) for d - [SDXS](https://github.com/IDKiro/sdxs) - [Hyper-SD](https://huggingface.co/ByteDance/Hyper-SD) - -Also supported are modifiers such as: -- **LCM**, **Turbo** and **Lightning** (*adversarial diffusion distillation*) networks -- All **LoRA** types such as LoCon, LyCORIS, HADA, IA3, Lokr, OFT -- **IP-Adapters** for SD 1.5 and SD-XL -- **InstantID**, **FaceSwap**, **FaceID**, **PhotoMerge** -- **AnimateDiff** for SD 1.5 -- **MuLAN** multi-language support - ## Platform support - *nVidia* GPUs using **CUDA** libraries on both *Windows and Linux* @@ -121,6 +97,25 @@ Also supported are modifiers such as: - Any GPU or device compatible with **OpenVINO** libraries on both *Windows and Linux* - *Apple M1/M2* on *OSX* using built-in support in Torch with **MPS** optimizations - *ONNX/Olive* +- *AMD* GPUs on Windows using **ZLUDA** libraries + +## Getting started + +- Get started with **SD.Next** by following the [installation instructions](wiki/Installation) +- For more details, check out [advanced installation](wiki/Advanced-Install) guide +- List and explanation of [command line arguments](wiki/CLI-Arguments) +- Install walkthrough [video](https://www.youtube.com/watch?v=nWTnTyFTuAs) + +> [!TIP] +> And for platform specific information, check out +> [WSL](wiki/WSL) | [Intel Arc](wiki/Intel-ARC) | [DirectML](wiki/DirectML) | [OpenVINO](wiki/OpenVINO) | [ONNX & Olive](wiki/ONNX-Runtime) | [ZLUDA](wiki/ZLUDA) | [AMD ROCm](wiki/AMD-ROCm) | [MacOS](wiki/MacOS-Python.md) | [nVidia](wiki/nVidia) + +> [!WARNING] +> If you run into issues, check out [troubleshooting](wiki/Troubleshooting) and [debugging](wiki/Debug) guides + +> [!TIP] +> All command line options can also be set via env variable +> For example `--debug` is same as `set SD_DEBUG=true` ## Backend support @@ -129,91 +124,11 @@ Also supported are modifiers such as: - **Diffusers**: Based on new [Huggingface Diffusers](https://huggingface.co/docs/diffusers/index) implementation Supports *all* models listed below This backend is set as default for new installations - See [wiki article](https://github.com/vladmandic/automatic/wiki/Diffusers) for more information - **Original**: Based on [LDM](https://github.com/Stability-AI/stablediffusion) reference implementation and significantly expanded on by [A1111](https://github.com/AUTOMATIC1111/stable-diffusion-webui) This backend and is fully compatible with most existing functionality and extensions written for *A1111 SDWebUI* Supports **SD 1.x** and **SD 2.x** models All other model types such as *SD-XL, LCM, Stable Cascade, PixArt, Playground, Segmind, Kandinsky, etc.* require backend **Diffusers** -## Examples - -*IP Adapters*: -![screenshot-ipadapter](https://github.com/user-attachments/assets/92830894-845c-49ec-92d9-18c8a577d04f) - -*Color grading*: -![screenshot-control](https://github.com/user-attachments/assets/cdad2722-ae7c-4c9c-94d6-5ea35a4b1356) - -*InstantID*: -![screenshot-instantid](https://github.com/user-attachments/assets/f38a5660-32b3-4235-9da1-c79eccf5372f) - -> [!IMPORTANT] -> - Loading any model other than standard SD 1.x / SD 2.x requires use of backend **Diffusers** -> - Loading any other models using **Original** backend is not supported -> - Loading manually download model `.safetensors` files is supported for specified models only (typically SD 1.x / SD 2.x / SD-XL models only) -> - For all other model types, use backend **Diffusers** and use built in Model downloader or - select model from Networks -> Models -> Reference list in which case it will be auto-downloaded and loaded - -## Install - -- [Step-by-step install guide](https://github.com/vladmandic/automatic/wiki/Installation) -- [Advanced install notes](https://github.com/vladmandic/automatic/wiki/Advanced-Install) -- [Video: install and use](https://www.youtube.com/watch?v=nWTnTyFTuAs) -- [Common installation errors](https://github.com/vladmandic/automatic/discussions/1627) -- [FAQ](https://github.com/vladmandic/automatic/discussions/1011) - -> [!TIP] -> - If you can't run SD.Next locally, try cloud deployment using [RunDiffusion](https://rundiffusion.com?utm_source=github&utm_medium=referral&utm_campaign=SDNext)! -> - Server can run with or without virtual environment, - Recommended to use `VENV` to avoid library version conflicts with other applications -> - **nVidia/CUDA** / **AMD/ROCm** / **Intel/OneAPI** are auto-detected if present and available, - For any other use case such as **DirectML**, **ONNX/Olive**, **OpenVINO** specify required parameter explicitly - or wrong packages may be installed as installer will assume CPU-only environment -> - Full startup sequence is logged in `sdnext.log`, - so if you encounter any issues, please check it first - -### Run - -Once SD.Next is installed, simply run `webui.ps1` or `webui.bat` (*Windows*) or `webui.sh` (*Linux or MacOS*) - -For list of available command line options, run `webui --help` for the full & up-to-date list - -> [!TIP] -> All command line options can also be set via env variable -> For example `--debug` is same as `set SD_DEBUG=true` - -## Notes - -> [!TIP] -> If you don't want to use built-in `venv` support and prefer to run SD.Next in your own environment such as *Docker* container, *Conda* environment or any other virtual environment, you can skip `venv` create/activate and launch SD.Next directly using `python launch.py` (command line flags noted above still apply). - -### Quantization - -**SD.Next** comes with broad quantization support, including support for BitsAndBytes, Optimum.Quanto, TorchAO, NNCF and GGUF -See [Quantization Wiki](https://github.com/vladmandic/automatic/wiki/Quantization) - -### Control - -**SD.Next** comes with built-in control for all types of text2image, image2image, video2video and batch processing - -*Control interface*: -![screenshot-control](https://github.com/user-attachments/assets/cdad2722-ae7c-4c9c-94d6-5ea35a4b1356) - -*Control processors*: -![screenshot-processors](https://github.com/user-attachments/assets/7bccb82b-366e-4bdb-ae57-cc53fac95d3c) - -*Masking*: -![screenshot-mask](https://github.com/user-attachments/assets/4b057e65-64f0-44ea-93b4-c3b69bc55532) - -### Extensions - -SD.Next comes with several extensions pre-installed: - -- [System Info](https://github.com/vladmandic/sd-extension-system-info) -- [chaiNNer](https://github.com/vladmandic/sd-extension-chainner) -- [RemBg](https://github.com/vladmandic/sd-extension-rembg) -- [Agent Scheduler](https://github.com/ArtVentureX/sd-webui-agent-scheduler) -- [Modern UI](https://github.com/BinaryQuantumSoul/sdnext-modernui) - ### Collab - We'd love to have additional maintainers (with comes with full repo rights). If you're interested, ping us! @@ -242,12 +157,6 @@ This should be fully cross-platform, but we'd really love to have additional con If you're unsure how to use a feature, best place to start is [Wiki](https://github.com/vladmandic/automatic/wiki) and if its not there, check [ChangeLog](CHANGELOG.md) for when feature was first introduced as it will always have a short note on how to use it -- [Wiki](https://github.com/vladmandic/automatic/wiki) -- [ReadMe](README.md) -- [ToDo](TODO.md) -- [ChangeLog](CHANGELOG.md) -- [CLI Tools](cli/README.md) - ### Sponsors
diff --git a/cli/README.md b/cli/README.md deleted file mode 100644 index 838db7a50..000000000 --- a/cli/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# Stable-Diffusion Productivity Scripts - -## API Examples - -### Run Generate - -- `cli/api-txt2img.py` -- `cli/api-img2img.py` -- `cli/api-control.py` - -### Monitor - -- `cli/api-progress.py` - -### Generic - -- `cli/api-json.py` - -### Process - -- `cli/api-info.py` -- `cli/api-upscale.py` -- `cli/api-vqa.py` -- `cli/api-preprocess.py` - -### Other - -- `cli/api-faceid.py` -- `cli/api-faces.py` -- `cli/api-mask.py` - -### JavaScript - -- `cli/api-txt2img.js` - -## Generate - -Text-to-image with all of the possible parameters -Supports upsampling, face restoration and grid creation -> python cli/generate.py - -By default uses parameters from `generate.json` - -Parameters that are not specified will be randomized: - -- Prompt will be dynamically created from template of random samples: `random.json` -- Sampler/Scheduler will be randomly picked from available ones -- CFG Scale set to 5-10 - -
- -## Auxiliary Scripts - -### Benchmark - -> python run-benchmark.py - -### Create Previews - -Create previews for **embeddings**, **lora**, **lycoris**, **dreambooth** and **hypernetwork** - -> python create-previews.py - -## Image Grid - -> python image-grid.py - -### Image Watermark - -Create invisible image watermark and remove existing EXIF tags - -> python image-watermark.py - -### Image Interrogate - -Runs CLiP and Booru image interrogation - -> python image-interrogate.py - -### Palette Extract - -Extract color palette from image(s) - -> python image-palette.py - -### Prompt Ideas - -Generate complex prompt ideas - -> python prompt-ideas.py - -### Prompt Promptist - -Attempts to beautify the provided prompt - -> python prompt-promptist.py - -### Video Extract - -Extract frames from video files - -> python video-extract.py - -
- -## Utility Scripts - -### SDAPI - -Utility module that handles async communication to Automatic API endpoints -Note: Requires SD API - -Can be used to manually execute specific commands: -> python sdapi.py progress -> python sdapi.py interrupt -> python sdapi.py shutdown diff --git a/modules/shared.py b/modules/shared.py index 171a777e9..48984c8be 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -473,15 +473,11 @@ options_templates.update(options_section(('sd', "Execution & Models"), { "sd_textencoder_cache": OptionInfo(True, "Cache text encoder results", gr.Checkbox, {"visible": False}), "sd_textencoder_cache_size": OptionInfo(4, "Text encoder results LRU cache size", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}), "stream_load": OptionInfo(False, "Load models using stream loading method", gr.Checkbox, {"visible": not native }), - "model_reuse_dict": OptionInfo(False, "Reuse loaded model dictionary", gr.Checkbox, {"visible": False}), "prompt_mean_norm": OptionInfo(False, "Prompt attention normalization", gr.Checkbox), "comma_padding_backtrack": OptionInfo(20, "Prompt padding", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1, "visible": not native }), "prompt_attention": OptionInfo("native", "Prompt attention parser", gr.Radio, {"choices": ["native", "compel", "xhinker", "a1111", "fixed"] }), "latent_history": OptionInfo(16, "Latent history size", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}), "sd_checkpoint_cache": OptionInfo(0, "Cached models", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1, "visible": not native }), - "sd_vae_checkpoint_cache": OptionInfo(0, "Cached VAEs", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1, "visible": False}), - "sd_disable_ckpt": OptionInfo(False, "Disallow models in ckpt format", gr.Checkbox, {"visible": False}), - "diffusers_version": OptionInfo("", "Diffusers version", gr.Textbox, {"visible": False}), })) options_templates.update(options_section(('cuda', "Compute Settings"), { @@ -495,8 +491,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "upcast_sampling": OptionInfo(False if sys.platform != "darwin" else True, "Upcast sampling"), "upcast_attn": OptionInfo(False, "Upcast attention layer"), "cuda_cast_unet": OptionInfo(False, "Fixed UNet precision"), - "disable_nan_check": OptionInfo(True, "Disable NaN check", gr.Checkbox, {"visible": False}), - "nan_skip": OptionInfo(False, "Skip Generation if NaN found in latents", gr.Checkbox, {"visible": True}), + "nan_skip": OptionInfo(False, "Skip Generation if NaN found in latents", gr.Checkbox), "rollback_vae": OptionInfo(False, "Attempt VAE roll back for NaN values"), "cross_attention_sep": OptionInfo("

Cross Attention

", "", gr.HTML), @@ -569,7 +564,6 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), { "diffusers_eval": OptionInfo(True, "Force model eval"), "diffusers_to_gpu": OptionInfo(False, "Load model directly to GPU"), "disable_accelerate": OptionInfo(False, "Disable accelerate"), - "diffusers_force_zeros": OptionInfo(False, "Force zeros for prompts when empty", gr.Checkbox, {"visible": False}), "diffusers_pooled": OptionInfo("default", "Diffusers SDXL pooled embeds", gr.Radio, {"choices": ['default', 'weighted']}), "diffusers_zeros_prompt_pad": OptionInfo(False, "Use zeros for prompt padding", gr.Checkbox), "huggingface_token": OptionInfo('', 'HuggingFace token'), @@ -650,9 +644,7 @@ options_templates.update(options_section(('system-paths', "System Paths"), { "vae_dir": OptionInfo(os.path.join(paths.models_path, 'VAE'), "Folder with VAE files", folder=True), "unet_dir": OptionInfo(os.path.join(paths.models_path, 'UNET'), "Folder with UNET files", folder=True), "te_dir": OptionInfo(os.path.join(paths.models_path, 'Text-encoder'), "Folder with Text encoder files", folder=True), - "sd_lora": OptionInfo("", "Add LoRA to prompt", gr.Textbox, {"visible": False}), "lora_dir": OptionInfo(os.path.join(paths.models_path, 'Lora'), "Folder with LoRA network(s)", folder=True), - "lyco_dir": OptionInfo(os.path.join(paths.models_path, 'LyCORIS'), "Folder with LyCORIS network(s)", gr.Text, {"visible": False}), "styles_dir": OptionInfo(os.path.join(paths.data_path, 'styles.csv'), "File or Folder with user-defined styles", folder=True), "wildcards_dir": OptionInfo(os.path.join(paths.models_path, 'wildcards'), "Folder with user-defined wildcards", folder=True), "embeddings_dir": OptionInfo(os.path.join(paths.models_path, 'embeddings'), "Folder with textual inversion embeddings", folder=True), @@ -728,12 +720,10 @@ options_templates.update(options_section(('saving-paths', "Image Naming & Paths" "saving_sep_images": OptionInfo("

Save options

", "", gr.HTML), "save_images_add_number": OptionInfo(True, "Numbered filenames", component_args=hide_dirs), "use_original_name_batch": OptionInfo(True, "Batch uses original name"), - "use_upscaler_name_as_suffix": OptionInfo(True, "Use upscaler as suffix", gr.Checkbox, {"visible": False}), "save_to_dirs": OptionInfo(False, "Save images to a subdirectory"), "directories_filename_pattern": OptionInfo("[date]", "Directory name pattern", component_args=hide_dirs), "samples_filename_pattern": OptionInfo("[seq]-[model_name]-[prompt_words]", "Images filename pattern", component_args=hide_dirs), "directories_max_prompt_words": OptionInfo(8, "Max words per pattern", gr.Slider, {"minimum": 1, "maximum": 99, "step": 1, **hide_dirs}), - "use_save_to_dirs_for_ui": OptionInfo(False, "Save images to a subdirectory when using Save button", gr.Checkbox, {"visible": False}), "outdir_sep_dirs": OptionInfo("

Folders

", "", gr.HTML), "outdir_samples": OptionInfo("", "Images folder", component_args=hide_dirs, folder=True), @@ -746,8 +736,6 @@ options_templates.update(options_section(('saving-paths', "Image Naming & Paths" "outdir_init_images": OptionInfo("outputs/init-images", "Folder for init images", component_args=hide_dirs, folder=True), "outdir_sep_grids": OptionInfo("

Grids

", "", gr.HTML), - "grid_extended_filename": OptionInfo(True, "Add extended info to filename when saving grid", gr.Checkbox, {"visible": False}), - "grid_save_to_dirs": OptionInfo(False, "Save grids to a subdirectory", gr.Checkbox, {"visible": False}), "outdir_grids": OptionInfo("", "Grids folder", component_args=hide_dirs, folder=True), "outdir_txt2img_grids": OptionInfo("outputs/grids", 'Folder for txt2img grids', component_args=hide_dirs, folder=True), "outdir_img2img_grids": OptionInfo("outputs/grids", 'Folder for img2img grids', component_args=hide_dirs, folder=True), @@ -760,7 +748,6 @@ options_templates.update(options_section(('ui', "User Interface Options"), { "gradio_theme": OptionInfo("black-teal", "UI theme", gr.Dropdown, lambda: {"choices": theme.list_themes()}, refresh=theme.refresh_themes), "autolaunch": OptionInfo(False, "Autolaunch browser upon startup"), "font_size": OptionInfo(14, "Font size", gr.Slider, {"minimum": 8, "maximum": 32, "step": 1, "visible": True}), - "tooltips": OptionInfo("UI Tooltips", "UI tooltips", gr.Radio, {"choices": ["None", "Browser default", "UI tooltips"], "visible": False}), "aspect_ratios": OptionInfo("1:1, 4:3, 3:2, 16:9, 16:10, 21:9, 2:3, 3:4, 9:16, 10:16, 9:21", "Allowed aspect ratios"), "motd": OptionInfo(True, "Show MOTD"), "compact_view": OptionInfo(False, "Compact view"), @@ -770,22 +757,14 @@ options_templates.update(options_section(('ui', "User Interface Options"), { "disable_weights_auto_swap": OptionInfo(True, "Do not change selected model when reading generation parameters"), "send_seed": OptionInfo(True, "Send seed when sending prompt or image to other interface"), "send_size": OptionInfo(True, "Send size when sending prompt or image to another interface"), - "keyedit_precision_attention": OptionInfo(0.1, "Ctrl+up/down precision when editing (attention:1.1)", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001, "visible": False}), - "keyedit_precision_extra": OptionInfo(0.05, "Ctrl+up/down precision when editing ", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001, "visible": False}), - "keyedit_delimiters": OptionInfo(r".,\/!?%^*;:{}=`~()", "Ctrl+up/down word delimiters", gr.Textbox, { "visible": False }), "quicksettings_list": OptionInfo(["sd_model_checkpoint"], "Quicksettings list", gr.Dropdown, lambda: {"multiselect":True, "choices": list(opts.data_labels.keys())}), - "ui_scripts_reorder": OptionInfo("", "UI scripts order", gr.Textbox, { "visible": False }), })) options_templates.update(options_section(('live-preview', "Live Previews"), { - "show_progressbar": OptionInfo(True, "Show progressbar", gr.Checkbox, {"visible": False}), - "live_previews_enable": OptionInfo(True, "Show live previews", gr.Checkbox, {"visible": False}), - "show_progress_grid": OptionInfo(True, "Show previews as a grid", gr.Checkbox, {"visible": False}), "notification_audio_enable": OptionInfo(False, "Play a notification upon completion"), "notification_audio_path": OptionInfo("html/notification.mp3","Path to notification sound", component_args=hide_dirs, folder=True), "show_progress_every_n_steps": OptionInfo(1, "Live preview display period", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1}), "show_progress_type": OptionInfo("Approximate", "Live preview method", gr.Radio, {"choices": ["Simple", "Approximate", "TAESD", "Full VAE"]}), - "live_preview_content": OptionInfo("Combined", "Live preview subject", gr.Radio, {"choices": ["Combined", "Prompt", "Negative prompt"], "visible": False}), "live_preview_refresh_period": OptionInfo(500, "Progress update period", gr.Slider, {"minimum": 0, "maximum": 5000, "step": 25}), "live_preview_taesd_layers": OptionInfo(3, "TAESD decode layers", gr.Slider, {"minimum": 1, "maximum": 3, "step": 1}), "logmonitor_show": OptionInfo(True, "Show log view"), @@ -832,8 +811,6 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"), 'uni_pc_variant': OptionInfo("bh2", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"], "visible": not native}), 'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"], "visible": not native}), "ddim_discretize": OptionInfo('uniform', "DDIM discretize img2img", gr.Radio, {"choices": ['uniform', 'quad'], "visible": not native}), - "pad_cond_uncond": OptionInfo(True, "Pad prompt and negative prompt to be same length", gr.Checkbox, {"visible": False}), - "batch_cond_uncond": OptionInfo(True, "Do conditional and unconditional denoising in one batch", gr.Checkbox, {"visible": False}), })) options_templates.update(options_section(('postprocessing', "Postprocessing"), { @@ -843,12 +820,10 @@ options_templates.update(options_section(('postprocessing', "Postprocessing"), { "postprocessing_sep_img2img": OptionInfo("

Img2Img & Inpainting

", "", gr.HTML), "img2img_color_correction": OptionInfo(False, "Apply color correction"), "mask_apply_overlay": OptionInfo(True, "Apply mask as overlay"), - "img2img_fix_steps": OptionInfo(False, "For image processing do exact number of steps as specified", gr.Checkbox, { "visible": False }), "img2img_background_color": OptionInfo("#ffffff", "Image transparent color fill", gr.ColorPicker, {}), "inpainting_mask_weight": OptionInfo(1.0, "Inpainting conditioning mask strength", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), "initial_noise_multiplier": OptionInfo(1.0, "Noise multiplier for image processing", gr.Slider, {"minimum": 0.1, "maximum": 1.5, "step": 0.01}), "img2img_extra_noise": OptionInfo(0.0, "Extra noise multiplier for img2img", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - "CLIP_stop_at_last_layers": OptionInfo(1, "Clip skip", gr.Slider, {"minimum": 1, "maximum": 8, "step": 1, "visible": False}), # "postprocessing_sep_detailer": OptionInfo("

Detailer

", "", gr.HTML), "detailer_model": OptionInfo("Detailer", "Detailer model", gr.Radio, lambda: {"choices": [x.name() for x in detailers], "visible": False}), @@ -870,7 +845,6 @@ options_templates.update(options_section(('postprocessing', "Postprocessing"), { "postprocessing_sep_upscalers": OptionInfo("

Upscaling

", "", gr.HTML), "upscaler_unload": OptionInfo(False, "Unload upscaler after processing"), - "upscaler_for_img2img": OptionInfo("None", "Default upscaler for image resize operations", gr.Dropdown, lambda: {"choices": [x.name for x in sd_upscalers], "visible": False}, refresh=refresh_upscalers), "upscaler_tile_size": OptionInfo(192, "Upscaler tile size", gr.Slider, {"minimum": 0, "maximum": 512, "step": 16}), "upscaler_tile_overlap": OptionInfo(8, "Upscaler tile overlap", gr.Slider, {"minimum": 0, "maximum": 64, "step": 1}), })) @@ -881,28 +855,12 @@ options_templates.update(options_section(('control', "Control Options"), { "control_unload_processor": OptionInfo(False, "Processor unload after use"), })) -options_templates.update(options_section(('interrogate', "Interrogate"), { # "Training" section disabled so just a placeholder - "unload_models_when_training": OptionInfo(False, "Move VAE and CLIP to RAM when training", gr.Checkbox, { "visible": False }), - "pin_memory": OptionInfo(True, "Pin training dataset to memory", gr.Checkbox, { "visible": False }), - "save_optimizer_state": OptionInfo(False, "Save resumable optimizer state when training", gr.Checkbox, { "visible": False }), - "save_training_settings_to_txt": OptionInfo(True, "Save training settings to a text file", gr.Checkbox, { "visible": False }), - "dataset_filename_word_regex": OptionInfo("", "Filename word regex", gr.Textbox, { "visible": False }), - "dataset_filename_join_string": OptionInfo(" ", "Filename join string", gr.Textbox, { "visible": False }), - "embeddings_templates_dir": OptionInfo("", "Embeddings train templates directory", gr.Textbox, { "visible": False }), - "training_image_repeats_per_epoch": OptionInfo(1, "Image repeats per epoch", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1, "visible": False }), - "training_write_csv_every": OptionInfo(0, "Save loss CSV file every n steps", gr.Number, { "visible": False }), - "training_enable_tensorboard": OptionInfo(False, "Enable tensorboard logging", gr.Checkbox, { "visible": False }), - "training_tensorboard_save_images": OptionInfo(False, "Save generated images within tensorboard", gr.Checkbox, { "visible": False }), - "training_tensorboard_flush_every": OptionInfo(120, "Tensorboard flush period", gr.Number, { "visible": False }), -})) - options_templates.update(options_section(('interrogate', "Interrogate"), { "interrogate_keep_models_in_memory": OptionInfo(False, "Interrogate: keep models in VRAM"), "interrogate_return_ranks": OptionInfo(True, "Interrogate: include ranks of model tags matches in results"), "interrogate_clip_num_beams": OptionInfo(1, "Interrogate: num_beams for BLIP", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1}), "interrogate_clip_min_length": OptionInfo(32, "Interrogate: minimum description length", gr.Slider, {"minimum": 1, "maximum": 128, "step": 1}), "interrogate_clip_max_length": OptionInfo(192, "Interrogate: maximum description length", gr.Slider, {"minimum": 1, "maximum": 256, "step": 1}), - "interrogate_clip_dict_limit": OptionInfo(2048, "CLIP: maximum number of lines in text file", gr.Slider, { "visible": False }), "interrogate_clip_skip_categories": OptionInfo(["artists", "movements", "flavors"], "Interrogate: skip categories", gr.CheckboxGroup, lambda: {"choices": modules.interrogate.category_types()}, refresh=modules.interrogate.category_types), "interrogate_deepbooru_score_threshold": OptionInfo(0.65, "Interrogate: deepbooru score threshold", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}), "deepbooru_sort_alpha": OptionInfo(False, "Interrogate: deepbooru sort alphabetically"), @@ -923,7 +881,6 @@ options_templates.update(options_section(('extra_networks', "Networks"), { "extra_networks_card_size": OptionInfo(160, "UI card size (px)", gr.Slider, {"minimum": 20, "maximum": 2000, "step": 1}), "extra_networks_card_square": OptionInfo(True, "UI disable variable aspect ratio"), "extra_networks_fetch": OptionInfo(True, "UI fetch network info on mouse-over"), - "extra_networks_card_fit": OptionInfo("cover", "UI image contain method", gr.Radio, {"choices": ["contain", "cover", "fill"], "visible": False}), "extra_network_skip_indexing": OptionInfo(False, "Build info on first access", gr.Checkbox), "extra_networks_model_sep": OptionInfo("

Models

", "", gr.HTML), @@ -944,17 +901,59 @@ options_templates.update(options_section(('extra_networks', "Networks"), { "lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 24, "step": 1}), "lora_quant": OptionInfo("NF4","LoRA precision in quantized models", gr.Radio, {"choices": ["NF4", "FP4"]}), - "lora_functional": OptionInfo(False, "Use Kohya method for handling multiple LoRA", gr.Checkbox, { "visible": False }), "lora_load_gpu": OptionInfo(True if not cmd_opts.lowvram else False, "Load LoRA directly to GPU"), +})) - "hypernetwork_enabled": OptionInfo(False, "Enable Hypernetwork support", gr.Checkbox, {"visible": False}), - "sd_hypernetwork": OptionInfo("None", "Add hypernetwork to prompt", gr.Dropdown, { "choices": ["None"], "visible": False }), +options_templates.update(options_section((None, "Internal options"), { + "diffusers_version": OptionInfo("", "Diffusers version", gr.Textbox, {"visible": False}), + "disabled_extensions": OptionInfo([], "Disable these extensions"), + "sd_checkpoint_hash": OptionInfo("", "SHA256 hash of the current checkpoint"), + "tooltips": OptionInfo("UI Tooltips", "UI tooltips", gr.Radio, {"choices": ["None", "Browser default", "UI tooltips"], "visible": False}), })) options_templates.update(options_section((None, "Hidden options"), { - "disabled_extensions": OptionInfo([], "Disable these extensions"), + "batch_cond_uncond": OptionInfo(True, "Do conditional and unconditional denoising in one batch", gr.Checkbox, {"visible": False}), + "CLIP_stop_at_last_layers": OptionInfo(1, "Clip skip", gr.Slider, {"minimum": 1, "maximum": 8, "step": 1, "visible": False}), + "dataset_filename_join_string": OptionInfo(" ", "Filename join string", gr.Textbox, { "visible": False }), + "dataset_filename_word_regex": OptionInfo("", "Filename word regex", gr.Textbox, { "visible": False }), + "diffusers_force_zeros": OptionInfo(False, "Force zeros for prompts when empty", gr.Checkbox, {"visible": False}), "disable_all_extensions": OptionInfo("none", "Disable all extensions (preserves the list of disabled extensions)", gr.Radio, {"choices": ["none", "user", "all"]}), - "sd_checkpoint_hash": OptionInfo("", "SHA256 hash of the current checkpoint"), + "disable_nan_check": OptionInfo(True, "Disable NaN check", gr.Checkbox, {"visible": False}), + "embeddings_templates_dir": OptionInfo("", "Embeddings train templates directory", gr.Textbox, { "visible": False }), + "extra_networks_card_fit": OptionInfo("cover", "UI image contain method", gr.Radio, {"choices": ["contain", "cover", "fill"], "visible": False}), + "grid_extended_filename": OptionInfo(True, "Add extended info to filename when saving grid", gr.Checkbox, {"visible": False}), + "grid_save_to_dirs": OptionInfo(False, "Save grids to a subdirectory", gr.Checkbox, {"visible": False}), + "hypernetwork_enabled": OptionInfo(False, "Enable Hypernetwork support", gr.Checkbox, {"visible": False}), + "img2img_fix_steps": OptionInfo(False, "For image processing do exact number of steps as specified", gr.Checkbox, { "visible": False }), + "interrogate_clip_dict_limit": OptionInfo(2048, "CLIP: maximum number of lines in text file", gr.Slider, { "visible": False }), + "keyedit_delimiters": OptionInfo(r".,\/!?%^*;:{}=`~()", "Ctrl+up/down word delimiters", gr.Textbox, { "visible": False }), + "keyedit_precision_attention": OptionInfo(0.1, "Ctrl+up/down precision when editing (attention:1.1)", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001, "visible": False}), + "keyedit_precision_extra": OptionInfo(0.05, "Ctrl+up/down precision when editing ", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001, "visible": False}), + "live_preview_content": OptionInfo("Combined", "Live preview subject", gr.Radio, {"choices": ["Combined", "Prompt", "Negative prompt"], "visible": False}), + "live_previews_enable": OptionInfo(True, "Show live previews", gr.Checkbox, {"visible": False}), + "lora_functional": OptionInfo(False, "Use Kohya method for handling multiple LoRA", gr.Checkbox, { "visible": False }), + "lyco_dir": OptionInfo(os.path.join(paths.models_path, 'LyCORIS'), "Folder with LyCORIS network(s)", gr.Text, {"visible": False}), + "model_reuse_dict": OptionInfo(False, "Reuse loaded model dictionary", gr.Checkbox, {"visible": False}), + "pad_cond_uncond": OptionInfo(True, "Pad prompt and negative prompt to be same length", gr.Checkbox, {"visible": False}), + "pin_memory": OptionInfo(True, "Pin training dataset to memory", gr.Checkbox, { "visible": False }), + "save_optimizer_state": OptionInfo(False, "Save resumable optimizer state when training", gr.Checkbox, { "visible": False }), + "save_training_settings_to_txt": OptionInfo(True, "Save training settings to a text file", gr.Checkbox, { "visible": False }), + "sd_disable_ckpt": OptionInfo(False, "Disallow models in ckpt format", gr.Checkbox, {"visible": False}), + "sd_hypernetwork": OptionInfo("None", "Add hypernetwork to prompt", gr.Dropdown, { "choices": ["None"], "visible": False }), + "sd_lora": OptionInfo("", "Add LoRA to prompt", gr.Textbox, {"visible": False}), + "sd_vae_checkpoint_cache": OptionInfo(0, "Cached VAEs", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1, "visible": False}), + "show_progress_grid": OptionInfo(True, "Show previews as a grid", gr.Checkbox, {"visible": False}), + "show_progressbar": OptionInfo(True, "Show progressbar", gr.Checkbox, {"visible": False}), + "training_enable_tensorboard": OptionInfo(False, "Enable tensorboard logging", gr.Checkbox, { "visible": False }), + "training_image_repeats_per_epoch": OptionInfo(1, "Image repeats per epoch", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1, "visible": False }), + "training_tensorboard_flush_every": OptionInfo(120, "Tensorboard flush period", gr.Number, { "visible": False }), + "training_tensorboard_save_images": OptionInfo(False, "Save generated images within tensorboard", gr.Checkbox, { "visible": False }), + "training_write_csv_every": OptionInfo(0, "Save loss CSV file every n steps", gr.Number, { "visible": False }), + "ui_scripts_reorder": OptionInfo("", "UI scripts order", gr.Textbox, { "visible": False }), + "unload_models_when_training": OptionInfo(False, "Move VAE and CLIP to RAM when training", gr.Checkbox, { "visible": False }), + "upscaler_for_img2img": OptionInfo("None", "Default upscaler for image resize operations", gr.Dropdown, lambda: {"choices": [x.name for x in sd_upscalers], "visible": False}, refresh=refresh_upscalers), + "use_save_to_dirs_for_ui": OptionInfo(False, "Save images to a subdirectory when using Save button", gr.Checkbox, {"visible": False}), + "use_upscaler_name_as_suffix": OptionInfo(True, "Use upscaler as suffix", gr.Checkbox, {"visible": False}), })) options_templates.update() From ba3a32ae470a371fdc59341abddc5304f22841c2 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 8 Nov 2024 09:49:46 -0500 Subject: [PATCH 26/40] add api override field Signed-off-by: Vladimir Mandic --- cli/api-txt2img.js | 19 +--------- modules/api/control.py | 2 + modules/api/generate.py | 4 ++ modules/api/models.py | 2 + modules/images.py | 84 +++++++++++++++++++++-------------------- 5 files changed, 53 insertions(+), 58 deletions(-) diff --git a/cli/api-txt2img.js b/cli/api-txt2img.js index 46d09b3a2..8d0e9f5d1 100755 --- a/cli/api-txt2img.js +++ b/cli/api-txt2img.js @@ -20,23 +20,6 @@ const sd_options = { cfg_scale: 6, width: 512, height: 512, - /* - // enable second pass - enable_hr: true, - // second pass: upscale - hr_upscaler: 'SCUNet GAN', - hr_scale: 2.0, - // second pass: hires - hr_force: true, - hr_second_pass_steps: 20, - hr_sampler_name: 'UniPC', - denoising_strength: 0.5, - // second pass: refiner - refiner_steps: 5, - refiner_start: 0.8, - refiner_prompt: '', - refiner_negative: '', - */ // api return options save_images: false, send_images: true, @@ -55,7 +38,7 @@ async function main() { const json = await res.json(); console.log('result:', json.info); for (const i in json.images) { // eslint-disable-line guard-for-in - const f = `/tmp/test-{${i}.jpg`; + const f = `/tmp/test-${i}.jpg`; fs.writeFileSync(f, atob(json.images[i]), 'binary'); console.log('image saved:', f); } diff --git a/modules/api/control.py b/modules/api/control.py index cf8916095..ffb000053 100644 --- a/modules/api/control.py +++ b/modules/api/control.py @@ -31,6 +31,7 @@ ReqControl = models.create_model_from_signature( {"key": "ip_adapter", "type": Optional[List[models.ItemIPAdapter]], "default": None, "exclude": True}, {"key": "face", "type": Optional[models.ItemFace], "default": None, "exclude": True}, {"key": "control", "type": Optional[List[ItemControl]], "default": [], "exclude": True}, + {"key": "extra", "type": Optional[dict], "default": {}, "exclude": True}, ] ) @@ -159,6 +160,7 @@ class APIControl(): output_processed = [] output_info = '' run.control_set({ 'do_not_save_grid': not req.save_images, 'do_not_save_samples': not req.save_images, **self.prepare_ip_adapter(req) }) + run.control_set(getattr(req, "extra", {})) res = run.control_run(**args) for item in res: if len(item) > 0 and (isinstance(item[0], list) or item[0] is None): # output_images diff --git a/modules/api/generate.py b/modules/api/generate.py index aeafa05a0..e22102057 100644 --- a/modules/api/generate.py +++ b/modules/api/generate.py @@ -106,6 +106,8 @@ class APIGenerate(): p.scripts = script_runner p.outpath_grids = shared.opts.outdir_grids or shared.opts.outdir_txt2img_grids p.outpath_samples = shared.opts.outdir_samples or shared.opts.outdir_txt2img_samples + for key, value in getattr(txt2imgreq, "extra", {}).items(): + setattr(p, key, value) shared.state.begin('API TXT', api=True) script_args = script.init_script_args(p, txt2imgreq, self.default_script_arg_txt2img, selectable_scripts, selectable_script_idx, script_runner) if selectable_scripts is not None: @@ -150,6 +152,8 @@ class APIGenerate(): p.scripts = script_runner p.outpath_grids = shared.opts.outdir_img2img_grids p.outpath_samples = shared.opts.outdir_img2img_samples + for key, value in getattr(img2imgreq, "extra", {}).items(): + setattr(p, key, value) shared.state.begin('API-IMG', api=True) script_args = script.init_script_args(p, img2imgreq, self.default_script_arg_img2img, selectable_scripts, selectable_script_idx, script_runner) if selectable_scripts is not None: diff --git a/modules/api/models.py b/modules/api/models.py index 3cf3aade9..740f3c555 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -210,6 +210,7 @@ ReqTxt2Img = PydanticModelGenerator( {"key": "alwayson_scripts", "type": dict, "default": {}}, {"key": "ip_adapter", "type": Optional[List[ItemIPAdapter]], "default": None, "exclude": True}, {"key": "face", "type": Optional[ItemFace], "default": None, "exclude": True}, + {"key": "extra", "type": Optional[dict], "default": {}, "exclude": True}, ] ).generate_model() StableDiffusionTxt2ImgProcessingAPI = ReqTxt2Img @@ -235,6 +236,7 @@ ReqImg2Img = PydanticModelGenerator( {"key": "alwayson_scripts", "type": dict, "default": {}}, {"key": "ip_adapter", "type": Optional[List[ItemIPAdapter]], "default": None, "exclude": True}, {"key": "face_id", "type": Optional[ItemFace], "default": None, "exclude": True}, + {"key": "extra", "type": Optional[dict], "default": {}, "exclude": True}, ] ).generate_model() StableDiffusionImg2ImgProcessingAPI = ReqImg2Img diff --git a/modules/images.py b/modules/images.py index fb9cc9652..910349bef 100644 --- a/modules/images.py +++ b/modules/images.py @@ -40,8 +40,6 @@ def atomically_save_image(): except Exception: shared.log.warning(f'Save: unknown image format: {extension}') image_format = 'JPEG' - if shared.opts.image_watermark_enabled or (shared.opts.image_watermark_position != 'none' and shared.opts.image_watermark_image != ''): - image = set_watermark(image, shared.opts.image_watermark) exifinfo = (exifinfo or "") if shared.opts.image_metadata else "" # additional metadata saved in files if shared.opts.save_txt and len(exifinfo) > 0: @@ -153,6 +151,11 @@ def save_image(image, info = image.info.get(pnginfo_section_name, '') if info is not None: pnginfo[pnginfo_section_name] = info + + wm_text = getattr(p, 'watermark_text', shared.opts.image_watermark) + wm_image = getattr(p, 'watermark_image', shared.opts.image_watermark_image) + image = set_watermark(image, wm_text, wm_image) + params = script_callbacks.ImageSaveParams(image, p, filename, pnginfo) params.filename = namegen.sanitize(filename) dirname = os.path.dirname(params.filename) @@ -369,45 +372,46 @@ def draw_overlay(im, text: str = '', y_offset: int = 0): return im -def set_watermark(image, watermark): - if shared.opts.image_watermark_position != 'none': # visible watermark - wm_image = None - try: - wm_image = Image.open(shared.opts.image_watermark_image) - if wm_image.mode != 'RGBA': - wm_image = wm_image.convert('RGBA') - except Exception as e: - shared.log.warning(f'Set image watermark: fn="{shared.opts.image_watermark_image}" {e}') - if wm_image is not None: - if shared.opts.image_watermark_position == 'top/left': - position = (0, 0) - elif shared.opts.image_watermark_position == 'top/right': - position = (image.width - wm_image.width, 0) - elif shared.opts.image_watermark_position == 'bottom/left': - position = (0, image.height - wm_image.height) - elif shared.opts.image_watermark_position == 'bottom/right': - position = (image.width - wm_image.width, image.height - wm_image.height) - elif shared.opts.image_watermark_position == 'center': - position = ((image.width - wm_image.width) // 2, (image.height - wm_image.height) // 2) - else: - position = (random.randint(0, image.width - wm_image.width), random.randint(0, image.height - wm_image.height)) +def set_watermark(image, wm_text: str = None, wm_image: Image.Image = None): + if shared.opts.image_watermark_position != 'none' and wm_image is not None: # visible watermark + if isinstance(wm_image, str): try: - for x in range(wm_image.width): - for y in range(wm_image.height): - rgba = wm_image.getpixel((x, y)) - orig = image.getpixel((x+position[0], y+position[1])) - # alpha blend - a = rgba[3] / 255 - r = int(rgba[0] * a + orig[0] * (1 - a)) - g = int(rgba[1] * a + orig[1] * (1 - a)) - b = int(rgba[2] * a + orig[2] * (1 - a)) - if not a == 0: - image.putpixel((x+position[0], y+position[1]), (r, g, b)) - shared.log.debug(f'Set image watermark: fn="{shared.opts.image_watermark_image}" image={wm_image} position={position}') + wm_image = Image.open(wm_image) except Exception as e: shared.log.warning(f'Set image watermark: image={wm_image} {e}') + return image + if isinstance(wm_image, Image.Image): + if wm_image.mode != 'RGBA': + wm_image = wm_image.convert('RGBA') + if shared.opts.image_watermark_position == 'top/left': + position = (0, 0) + elif shared.opts.image_watermark_position == 'top/right': + position = (image.width - wm_image.width, 0) + elif shared.opts.image_watermark_position == 'bottom/left': + position = (0, image.height - wm_image.height) + elif shared.opts.image_watermark_position == 'bottom/right': + position = (image.width - wm_image.width, image.height - wm_image.height) + elif shared.opts.image_watermark_position == 'center': + position = ((image.width - wm_image.width) // 2, (image.height - wm_image.height) // 2) + else: + position = (random.randint(0, image.width - wm_image.width), random.randint(0, image.height - wm_image.height)) + try: + for x in range(wm_image.width): + for y in range(wm_image.height): + rgba = wm_image.getpixel((x, y)) + orig = image.getpixel((x+position[0], y+position[1])) + # alpha blend + a = rgba[3] / 255 + r = int(rgba[0] * a + orig[0] * (1 - a)) + g = int(rgba[1] * a + orig[1] * (1 - a)) + b = int(rgba[2] * a + orig[2] * (1 - a)) + if not a == 0: + image.putpixel((x+position[0], y+position[1]), (r, g, b)) + shared.log.debug(f'Set image watermark: image={wm_image} position={position}') + except Exception as e: + shared.log.warning(f'Set image watermark: image={wm_image} {e}') - if shared.opts.image_watermark_enabled: # invisible watermark + if shared.opts.image_watermark_enabled and wm_text is not None: # invisible watermark from imwatermark import WatermarkEncoder wm_type = 'bytes' wm_method = 'dwtDctSvd' @@ -416,16 +420,16 @@ def set_watermark(image, watermark): info = image.info data = np.asarray(image) encoder = WatermarkEncoder() - text = f"{watermark:<{length}}"[:length] + text = f"{wm_text:<{length}}"[:length] bytearr = text.encode(encoding='ascii', errors='ignore') try: encoder.set_watermark(wm_type, bytearr) encoded = encoder.encode(data, wm_method) image = Image.fromarray(encoded) image.info = info - shared.log.debug(f'Set invisible watermark: {watermark} method={wm_method} bits={wm_length}') + shared.log.debug(f'Set invisible watermark: {wm_text} method={wm_method} bits={wm_length}') except Exception as e: - shared.log.warning(f'Set invisible watermark error: {watermark} method={wm_method} bits={wm_length} {e}') + shared.log.warning(f'Set invisible watermark error: {wm_text} method={wm_method} bits={wm_length} {e}') return image From 5de457bb18dbf2d4564e50840b44c2ca576066a3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 8 Nov 2024 11:32:47 -0500 Subject: [PATCH 27/40] sort and describe all scripts Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 +++ javascript/extraNetworks.js | 4 +++- modules/face/__init__.py | 4 ++-- modules/scripts.py | 4 +++- modules/ui_common.py | 2 +- scripts/animatediff.py | 2 +- scripts/apg.py | 4 ++-- scripts/blipdiffusion.py | 11 ++++------- scripts/cogvideo.py | 2 +- scripts/consistory_ext.py | 2 +- scripts/ctrlx.py | 4 ++-- scripts/demofusion.py | 4 ++-- scripts/differential_diffusion.py | 4 ++-- scripts/hdr.py | 4 ++-- scripts/image2video.py | 3 ++- scripts/instantir.py | 2 +- scripts/k_diff.py | 4 ++-- scripts/layerdiffuse.py | 4 ++-- scripts/ledits.py | 4 ++-- scripts/lut.py | 2 +- scripts/mixture_tiling.py | 4 ++-- scripts/mulan.py | 4 ++-- scripts/pulid_ext.py | 2 +- scripts/resadapter.py | 4 ++-- scripts/sd_upscale.py | 2 +- scripts/stablevideodiffusion.py | 2 +- scripts/t_gate.py | 4 ++-- scripts/text2video.py | 2 +- 28 files changed, 51 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d78943f42..b989132f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ This release can be considered an LTS release before we kick off the next round - create video from generated grid images supports all standard video types and interpolation - UI: + - better gallery and networks sidebar sizing - add additional [hotkeys](https://github.com/vladmandic/automatic/wiki/Hotkeys) - add show networks on startup setting - better mapping of networks previews @@ -68,6 +69,8 @@ This release can be considered an LTS release before we kick off the next round - Auto-remove invalid packages from `venv/site-packages` e.g. packages starting with `~` which are left-over due to windows access violation - Requirements: update + - Scripts: + - More verbose descriptions for all scripts - Model loader: - Report modules included in safetensors when attempting to load a model - CLI: diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 9a33baa86..77fe125f3 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -481,11 +481,13 @@ function setupExtraNetworksForTab(tabname) { en.style.position = 'absolute'; en.style.height = 'auto'; en.style.width = `${window.opts.extra_networks_sidebar_width}vw`; + en.style.maxWidth = '655px'; en.style.right = '0'; en.style.top = '13em'; en.style.transition = 'width 0.3s ease'; en.style.zIndex = 100; - gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = `${100 - 2 - window.opts.extra_networks_sidebar_width}vw`; + // gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = `${100 - 2 - window.opts.extra_networks_sidebar_width}vw`; + gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = `calc(100vw - 2em - min(${window.opts.extra_networks_sidebar_width}vw, 655px))`; } else { en.style.position = 'relative'; en.style.height = 'unset'; diff --git a/modules/face/__init__.py b/modules/face/__init__.py index aae3cdaa2..19af6d01b 100644 --- a/modules/face/__init__.py +++ b/modules/face/__init__.py @@ -9,7 +9,7 @@ debug = shared.log.trace if os.environ.get('SD_FACE_DEBUG', None) is not None el class Script(scripts.Script): def title(self): - return 'Face' + return 'Face: Multiple ID Transfers' def show(self, is_img2img): return True if shared.native else False @@ -45,7 +45,7 @@ class Script(scripts.Script): # return signature is array of gradio components def ui(self, _is_img2img): with gr.Row(): - gr.HTML("  Face module
") + gr.HTML("  Face: Multiple ID Transfers
") with gr.Row(): mode = gr.Dropdown(label='Mode', choices=['None', 'FaceID', 'FaceSwap', 'InstantID', 'PhotoMaker'], value='None') with gr.Group(visible=False) as cfg_faceid: diff --git a/modules/scripts.py b/modules/scripts.py index 8a67d0a50..cf2cf25b9 100644 --- a/modules/scripts.py +++ b/modules/scripts.py @@ -352,7 +352,9 @@ class ScriptRunner: self.selectable_scripts.clear() auto_processing_scripts = scripts_auto_postprocessing.create_auto_preprocessing_script_data() - for script_class, path, _basedir, _script_module in auto_processing_scripts + scripts_data: + all_scripts = auto_processing_scripts + scripts_data + sorted_scripts = sorted(all_scripts, key=lambda x: x.script_class().title().lower()) + for script_class, path, _basedir, _script_module in sorted_scripts: try: script = script_class() script.filename = path diff --git a/modules/ui_common.py b/modules/ui_common.py index 9ad87c17f..9c4bb5cdc 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -246,7 +246,7 @@ def create_output_panel(tabname, preview=True, prompt=None, height=None): # columns are for <576px, <768px, <992px, <1200px, <1400px, >1400px result_gallery = gr.Gallery(value=[], label='Output', show_label=False, show_download_button=True, allow_preview=True, container=False, preview=preview, - columns=5, object_fit='scale-down', height=height, + columns=4, object_fit='scale-down', height=height, elem_id=f"{tabname}_gallery", ) if prompt is not None: diff --git a/scripts/animatediff.py b/scripts/animatediff.py index fca09424b..4c50f9cf6 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -189,7 +189,7 @@ def set_free_noise(frames): class Script(scripts.Script): def title(self): - return 'AnimateDiff' + return 'Video AnimateDiff' def show(self, is_img2img): # return scripts.AlwaysVisible if shared.native else False diff --git a/scripts/apg.py b/scripts/apg.py index 6a3020c38..0476da246 100644 --- a/scripts/apg.py +++ b/scripts/apg.py @@ -9,14 +9,14 @@ class Script(scripts.Script): self.register() def title(self): - return 'APG' + return 'APG: Adaptive Projected Guidance' def show(self, is_img2img): return not is_img2img if shared.native else False def ui(self, _is_img2img): # ui elements with gr.Row(): - gr.HTML('  APG: Adaptive projected guidance
') + gr.HTML('  APG: Adaptive Projected Guidance
') with gr.Row(): eta = gr.Slider(label="ETA", value=1.0, minimum=0, maximum=2.0, step=0.05) momentum = gr.Slider(label="Momentum", value=-0.50, minimum=-1.0, maximum=1.0, step=0.05) diff --git a/scripts/blipdiffusion.py b/scripts/blipdiffusion.py index 39d8974e1..0acc80929 100644 --- a/scripts/blipdiffusion.py +++ b/scripts/blipdiffusion.py @@ -2,19 +2,16 @@ import gradio as gr from modules import scripts, processing, shared, sd_models -title = 'BLIP Diffusion' - - class Script(scripts.Script): def title(self): - return title + return 'BLIP Diffusion: Controllable Generation and Editing' def show(self, is_img2img): return is_img2img if shared.native else False def ui(self, _is_img2img): with gr.Row(): - gr.HTML('  BLIP Diffusion
') + gr.HTML('  BLIP Diffusion: Controllable Generation and Editing
') with gr.Row(): source_subject = gr.Textbox(value='', label='Source subject') with gr.Row(): @@ -26,7 +23,7 @@ class Script(scripts.Script): def run(self, p: processing.StableDiffusionProcessing, source_subject, target_subject, prompt_strength): # pylint: disable=arguments-differ, unused-argument c = shared.sd_model.__class__.__name__ if shared.sd_loaded else '' if c != 'BlipDiffusionPipeline': - shared.log.error(f'{title}: model selected={c} required=BLIPDiffusion') + shared.log.error(f'BLIP: model selected={c} required=BLIPDiffusion') return None if hasattr(p, 'init_images') and len(p.init_images) > 0: p.task_args['reference_image'] = p.init_images[0] @@ -41,5 +38,5 @@ class Script(scripts.Script): processed = processing.process_images(p) return processed else: - shared.log.error(f'{title}: no init_images') + shared.log.error('BLIP: no init_images') return None diff --git a/scripts/cogvideo.py b/scripts/cogvideo.py index a4a3141d4..7f2c7225e 100644 --- a/scripts/cogvideo.py +++ b/scripts/cogvideo.py @@ -22,7 +22,7 @@ debug = (os.environ.get('SD_LOAD_DEBUG', None) is not None) or (os.environ.get(' class Script(scripts.Script): def title(self): - return 'CogVideoX' + return 'Video CogVideoX' def show(self, is_img2img): return shared.native diff --git a/scripts/consistory_ext.py b/scripts/consistory_ext.py index b7454aa75..45de1ea6c 100644 --- a/scripts/consistory_ext.py +++ b/scripts/consistory_ext.py @@ -22,7 +22,7 @@ class Script(scripts.Script): self.anchor_cache_second_stage = None def title(self): - return 'ConsiStory' + return 'ConsiStory: Consistent Image Generation' def show(self, is_img2img): return not is_img2img if shared.native and shared.cmd_opts.experimental else False diff --git a/scripts/ctrlx.py b/scripts/ctrlx.py index 69d5994df..f372e2a94 100644 --- a/scripts/ctrlx.py +++ b/scripts/ctrlx.py @@ -7,14 +7,14 @@ from modules import shared, scripts, processing, processing_helpers, sd_models, class Script(scripts.Script): def title(self): - return 'Ctrl-X' + return 'Ctrl-X: Controlling Structure and Appearance' def show(self, is_img2img): return shared.native def ui(self, _is_img2img): with gr.Row(): - gr.HTML('  Ctrl-X
') + gr.HTML('  Ctrl-X: Controlling Structure and Appearance
') with gr.Accordion(label='Structure', open=True): with gr.Row(): struct_prompt = gr.Textbox(label='Prompt', value='', rows=1) diff --git a/scripts/demofusion.py b/scripts/demofusion.py index f7cdfe543..6625c0c79 100644 --- a/scripts/demofusion.py +++ b/scripts/demofusion.py @@ -1221,7 +1221,7 @@ class DemoFusionSDXLPipeline(DiffusionPipeline, FromSingleFileMixin, LoraLoaderM class Script(scripts.Script): def title(self): - return 'DemoFusion' + return 'DemoFusion: High-Resolution Image Generation' def show(self, is_img2img): return not is_img2img if shared.native else False @@ -1229,7 +1229,7 @@ class Script(scripts.Script): # return signature is array of gradio components def ui(self, _is_img2img): with gr.Row(): - gr.HTML('  DemoFusion
') + gr.HTML('  DemoFusion: High-Resolution Image Generation
') with gr.Row(): cosine_scale_1 = gr.Slider(minimum=0, maximum=5, step=0.1, value=3, label="Cosine scale 1") cosine_scale_2 = gr.Slider(minimum=0, maximum=5, step=0.1, value=1, label="Cosine scale 2") diff --git a/scripts/differential_diffusion.py b/scripts/differential_diffusion.py index 705242987..da4ae0e2e 100644 --- a/scripts/differential_diffusion.py +++ b/scripts/differential_diffusion.py @@ -1858,14 +1858,14 @@ MODELS = { class Script(scripts.Script): def title(self): - return 'Differential diffusion' + return 'Differential diffusion: Individual Pixel Strength' def show(self, is_img2img): return is_img2img if shared.native else False def ui(self, _is_img2img): with gr.Row(): - gr.HTML('  Differential diffusion
Select a model for auto-preprocess or upload an image map
') + gr.HTML('  Differential diffusion: Individual Pixel Strength
Select a model for auto-preprocess or upload an image map
') with gr.Row(): enabled = gr.Checkbox(label='Enabled', value=True) invert = gr.Checkbox(label='Mask invert', value=False) diff --git a/scripts/hdr.py b/scripts/hdr.py index 788c0add2..9afc3673b 100644 --- a/scripts/hdr.py +++ b/scripts/hdr.py @@ -11,14 +11,14 @@ from modules.shared import opts, state class Script(scripts.Script): def title(self): - return "HDR" + return "HDR: High Dynamic Range" def show(self, is_img2img): return True def ui(self, is_img2img): with gr.Row(): - gr.HTML("  High Dynamic Range
") + gr.HTML("  HDR: High Dynamic Range
") with gr.Row(): save_hdr = gr.Checkbox(label="Save HDR image", value=True) hdr_range = gr.Slider(minimum=0, maximum=1, step=0.05, value=0.65, label='HDR range') diff --git a/scripts/image2video.py b/scripts/image2video.py index 332972a6d..876ed3193 100644 --- a/scripts/image2video.py +++ b/scripts/image2video.py @@ -13,7 +13,7 @@ MODELS = [ class Script(scripts.Script): def title(self): - return 'Image-to-Video' + return 'Video VGen Image-to-Video' def show(self, is_img2img): return is_img2img if shared.native else False @@ -102,6 +102,7 @@ class Script(scripts.Script): processed = processing.process_images(p) shared.sd_model.motion_adapter = None + processed = None if model_name == 'VGen': if not isinstance(shared.sd_model, diffusers.I2VGenXLPipeline): shared.log.info(f'Image2Video VGen load: model={repo_id}') diff --git a/scripts/instantir.py b/scripts/instantir.py index 4c7ce77b7..5eb7d503a 100644 --- a/scripts/instantir.py +++ b/scripts/instantir.py @@ -12,7 +12,7 @@ class Script(scripts.Script): self.orig_ip_unapply = None def title(self): - return 'InstantIR' + return 'InstantIR: Image Restoration' def show(self, is_img2img): return is_img2img if shared.native else False diff --git a/scripts/k_diff.py b/scripts/k_diff.py index 354df5d4b..92b43149d 100644 --- a/scripts/k_diff.py +++ b/scripts/k_diff.py @@ -9,14 +9,14 @@ class Script(scripts.Script): orig_pipe = None def title(self): - return 'K-Diffusion' + return 'K-Diffusion Samplers' def show(self, is_img2img): return not is_img2img if shared.native else False def ui(self, _is_img2img): # ui elements with gr.Row(): - gr.HTML('  K-Diffusion samplers
') + gr.HTML('  K-Diffusion Samplers
') with gr.Row(): sampler = gr.Dropdown(label="Sampler", choices=self.samplers()) return [sampler] diff --git a/scripts/layerdiffuse.py b/scripts/layerdiffuse.py index a1e15aa8b..ecf7da1d3 100644 --- a/scripts/layerdiffuse.py +++ b/scripts/layerdiffuse.py @@ -5,7 +5,7 @@ from modules import shared, scripts, sd_models class Script(scripts.Script): def title(self): - return 'LayerDiffuse' + return 'LayerDiffuse: Transparent Image' def show(self, is_img2img): return True if shared.native else False @@ -40,7 +40,7 @@ class Script(scripts.Script): def ui(self, _is_img2img): with gr.Row(): gr.HTML(""" -   LayerDiffuse

+   LayerDiffuse: Transparent Image

- Click Apply to model to apply LayerDiffuse to current model
- Click Reload model to remove LayerDiffuse from current model

""") diff --git a/scripts/ledits.py b/scripts/ledits.py index ba9d49f89..b75c6ff6f 100644 --- a/scripts/ledits.py +++ b/scripts/ledits.py @@ -5,7 +5,7 @@ from modules import scripts, processing, shared, devices, sd_models class Script(scripts.Script): def title(self): - return 'LEdits++' + return 'LEdits: Limitless Image Editing' def show(self, is_img2img): return is_img2img if shared.native else False @@ -13,7 +13,7 @@ class Script(scripts.Script): # return signature is array of gradio components def ui(self, _is_img2img): with gr.Row(): - gr.HTML('  LEdits++
') + gr.HTML('  LEdits++: Limitless Image Editing
') with gr.Row(): edit_start = gr.Slider(label='Edit start', minimum=0.0, maximum=1.0, step=0.01, value=0.1) edit_stop = gr.Slider(label='Edit stop', minimum=0.0, maximum=1.0, step=0.01, value=1.0) diff --git a/scripts/lut.py b/scripts/lut.py index 3d240f291..573222161 100644 --- a/scripts/lut.py +++ b/scripts/lut.py @@ -17,7 +17,7 @@ class Script(scripts.Script): def ui(self, _is_img2img): with gr.Row(): - gr.HTML("  Color grading
") + gr.HTML("  LUT Color grading
") with gr.Row(): original = gr.Checkbox(label='Include original image', value=True) with gr.Row(): diff --git a/scripts/mixture_tiling.py b/scripts/mixture_tiling.py index 13e48ce11..5b5aab9db 100644 --- a/scripts/mixture_tiling.py +++ b/scripts/mixture_tiling.py @@ -26,14 +26,14 @@ def check_dependencies(): class Script(scripts.Script): def title(self): - return 'Mixture tiling' + return 'Mixture Tiling: Scene Composition' def show(self, is_img2img): return not is_img2img if shared.native else False def ui(self, _is_img2img): with gr.Row(): - gr.HTML('  Mixture tiling
') + gr.HTML('  Mixture Tiling: Scene Composition
') with gr.Row(): gr.HTML('  Separated prompts using new lines
  Number of prompts must matcxh X*Y') with gr.Row(): diff --git a/scripts/mulan.py b/scripts/mulan.py index c2ad10d2e..829ce1463 100644 --- a/scripts/mulan.py +++ b/scripts/mulan.py @@ -46,14 +46,14 @@ text_encoder_path = None class Script(scripts.Script): def title(self): - return 'MuLan' + return 'MuLan: Multi Language Prompts' def show(self, is_img2img): return True if shared.native else False def ui(self, _is_img2img): with gr.Row(): - gr.HTML('  MuLan
') + gr.HTML('  MuLan: Multi Language Prompts
') with gr.Row(): selected_encoder = gr.Dropdown(label='Encoder', choices=ENCODERS, value=ENCODERS[0]) return [selected_encoder] diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 5d209c211..b7fad31bc 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -20,7 +20,7 @@ class Script(scripts.Script): self.register() # pulid is script with processing override so xyz doesnt execute def title(self): - return 'PuLID' + return 'PuLID: ID Customization' def show(self, _is_img2img): return shared.native diff --git a/scripts/resadapter.py b/scripts/resadapter.py index cbd0bf671..58162f9ab 100644 --- a/scripts/resadapter.py +++ b/scripts/resadapter.py @@ -19,7 +19,7 @@ models = { class Script(scripts.Script): def title(self): - return 'ResAdapter' + return 'ResAdapter: Domain Consistent Resolution' def show(self, is_img2img): return not is_img2img if shared.native else False @@ -27,7 +27,7 @@ class Script(scripts.Script): # return signature is array of gradio components def ui(self, _is_img2img): with gr.Row(): - gr.HTML('  ResAdapter
') + gr.HTML('  ResAdapter: Domain Consistent Resolution
') with gr.Row(): model = gr.Dropdown(label="Model", choices=list(models), value="None") weight = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label="Weight", value=1.0) diff --git a/scripts/sd_upscale.py b/scripts/sd_upscale.py index 9f21c5645..9c5a72204 100644 --- a/scripts/sd_upscale.py +++ b/scripts/sd_upscale.py @@ -9,7 +9,7 @@ from modules.shared import opts, state, log class Script(scripts.Script): def title(self): - return "SD upscale" + return "SD Upscale" def show(self, is_img2img): return is_img2img diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index 585871edc..cbf2ce003 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -16,7 +16,7 @@ models = { class Script(scripts.Script): def title(self): - return 'Stable Video Diffusion' + return 'Video: SVD' def show(self, is_img2img): return is_img2img if shared.native else False diff --git a/scripts/t_gate.py b/scripts/t_gate.py index 3bd51445d..3808a796d 100644 --- a/scripts/t_gate.py +++ b/scripts/t_gate.py @@ -5,7 +5,7 @@ from installer import install class Script(scripts.Script): def title(self): - return 'T-Gate' + return 'T-Gate: Accelerate via Gating Attention' def show(self, is_img2img): return not is_img2img if shared.native else False @@ -13,7 +13,7 @@ class Script(scripts.Script): # return signature is array of gradio components def ui(self, _is_img2img): with gr.Row(): - gr.HTML('  T-Gate
') + gr.HTML('  T-Gate: Accelerate via Gating Attention
') with gr.Row(): enabled = gr.Checkbox(label="Enabled", value=True) with gr.Row(): diff --git a/scripts/text2video.py b/scripts/text2video.py index 8dec9bd0e..dc4c44cac 100644 --- a/scripts/text2video.py +++ b/scripts/text2video.py @@ -23,7 +23,7 @@ MODELS = [ class Script(scripts.Script): def title(self): - return 'Text-to-Video' + return 'Video: ModelScope' def show(self, is_img2img): return not is_img2img if shared.native else False From 34bc7377d86def0a635265c662a4f0d8f622bd2f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 8 Nov 2024 14:00:50 -0500 Subject: [PATCH 28/40] update pulid Signed-off-by: Vladimir Mandic --- modules/processing_class.py | 2 +- modules/pulid/pulid_sampling.py | 12 ++++++------ modules/sd_models.py | 9 +++++++-- scripts/apg.py | 7 +++++++ scripts/pulid_ext.py | 29 +++++++++++++++++++---------- scripts/xyz_grid_classes.py | 1 + 6 files changed, 41 insertions(+), 19 deletions(-) diff --git a/modules/processing_class.py b/modules/processing_class.py index d38aae790..2f11db375 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -486,7 +486,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): image = images.resize_image(self.resize_mode, image, self.width, self.height, upscaler_name=self.resize_name, context=self.resize_context) self.width = image.width self.height = image.height - if self.image_mask is not None and shared.opts.mask_apply_overlay: + if self.image_mask is not None and shared.opts.mask_apply_overlay and not hasattr(self, 'xyz'): image_masked = Image.new('RGBa', (image.width, image.height)) image_to_paste = image.convert("RGBA").convert("RGBa") image_to_mask = ImageOps.invert(self.mask_for_overlay.convert('L')) if self.mask_for_overlay is not None else None diff --git a/modules/pulid/pulid_sampling.py b/modules/pulid/pulid_sampling.py index e319c0d27..6a2ef31f3 100644 --- a/modules/pulid/pulid_sampling.py +++ b/modules/pulid/pulid_sampling.py @@ -393,8 +393,8 @@ def sample_dpmpp_2s_ancestral(model, x, sigmas, extra_args=None, callback=None, extra_args = {} if extra_args is None else extra_args noise_sampler = default_noise_sampler(x) if noise_sampler is None else noise_sampler s_in = x.new_ones([x.shape[0]]) - sigma_fn = lambda t: t.neg().exp() - t_fn = lambda sigma: sigma.log().neg() + sigma_fn = lambda t: t.neg().exp() # pylint: disable=C3001 + t_fn = lambda sigma: sigma.log().neg() # pylint: disable=C3001 for i in trange(len(sigmas) - 1, disable=disable): denoised = model(x, sigmas[i] * s_in, **extra_args) @@ -430,8 +430,8 @@ def sample_dpmpp_sde(model, x, sigmas, extra_args=None, callback=None, disable=N noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max) if noise_sampler is None else noise_sampler extra_args = {} if extra_args is None else extra_args s_in = x.new_ones([x.shape[0]]) - sigma_fn = lambda t: t.neg().exp() - t_fn = lambda sigma: sigma.log().neg() + sigma_fn = lambda t: t.neg().exp() # pylint: disable=C3001 + t_fn = lambda sigma: sigma.log().neg() # pylint: disable=C3001 for i in trange(len(sigmas) - 1, disable=disable): denoised = model(x, sigmas[i] * s_in, **extra_args) @@ -472,8 +472,8 @@ def sample_dpmpp_2m(model, x, sigmas, extra_args=None, callback=None, disable=No """DPM-Solver++(2M).""" extra_args = {} if extra_args is None else extra_args s_in = x.new_ones([x.shape[0]]) - sigma_fn = lambda t: t.neg().exp() - t_fn = lambda sigma: sigma.log().neg() + sigma_fn = lambda t: t.neg().exp() # pylint: disable=C3001 + t_fn = lambda sigma: sigma.log().neg() # pylint: disable=C3001 old_denoised = None for i in trange(len(sigmas) - 1, disable=disable): diff --git a/modules/sd_models.py b/modules/sd_models.py index 1f932bd30..8320ac293 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -449,6 +449,9 @@ def move_model(model, device=None, force=False): devices.torch_gc() return + if hasattr(model, 'pipe'): + move_model(model.pipe, device, force) + fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access if getattr(model, 'vae', None) is not None and get_diffusers_task(model) != DiffusersTaskType.TEXT_2_IMAGE: if device == devices.device and model.vae.device.type != "meta": # force vae back to gpu if not in txt2img mode @@ -476,7 +479,8 @@ def move_model(model, device=None, force=False): try: t0 = time.time() try: - model.to(device) + if hasattr(model, 'to'): + model.to(device) if hasattr(model, "prior_pipe"): model.prior_pipe.to(device) except Exception as e0: @@ -486,7 +490,8 @@ def move_model(model, device=None, force=False): if hasattr(component, 'modules'): for module in component.modules(): try: - module.to(device) + if hasattr(module, 'to'): + module.to(device) except Exception as e2: if 'Cannot copy out of meta tensor' in str(e2): if os.environ.get('SD_MOVE_DEBUG', None): diff --git a/scripts/apg.py b/scripts/apg.py index 0476da246..6d0ec107e 100644 --- a/scripts/apg.py +++ b/scripts/apg.py @@ -2,6 +2,9 @@ import gradio as gr from modules import scripts, processing, shared, sd_models +registered = False + + class Script(scripts.Script): def __init__(self): super().__init__() @@ -24,6 +27,10 @@ class Script(scripts.Script): return [eta, momentum, threshold] def register(self): # register xyz grid elements + global registered # pylint: disable=global-statement + if registered: + return + registered = True def apply_field(field): def fun(p, x, xs): # pylint: disable=unused-argument setattr(p, field, x) diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index b7fad31bc..7ec32e904 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -9,13 +9,15 @@ from modules import shared, devices, errors, scripts, processing, processing_hel debug = os.environ.get('SD_PULID_DEBUG', None) is not None direct = False +registered = False +uploaded_images = [] class Script(scripts.Script): def __init__(self): - self.images = [] self.pulid = None self.cache = None + self.mask_apply_overlay = shared.opts.mask_apply_overlay super().__init__() self.register() # pulid is script with processing override so xyz doesnt execute @@ -33,6 +35,10 @@ class Script(scripts.Script): install('pydantic==1.10.15', 'pydantic', ignore=False, reinstall=True) def register(self): # register xyz grid elements + global registered # pylint: disable=global-statement + if registered: + return + registered = True def apply_field(field): def fun(p, x, xs): # pylint: disable=unused-argument setattr(p, field, x) @@ -52,7 +58,7 @@ class Script(scripts.Script): def load_images(self, files): - self.images = [] + uploaded_images.clear() for file in files or []: try: if isinstance(file, str): @@ -66,10 +72,10 @@ class Script(scripts.Script): image = Image.open(file.name) # _TemporaryFileWrapper from gr.Files else: raise ValueError(f'IP adapter unknown input: {file}') - self.images.append(image) + uploaded_images.append(image) except Exception as e: shared.log.warning(f'IP adapter failed to load image: {e}') - return gr.update(value=self.images, visible=len(self.images) > 0) + return gr.update(value=uploaded_images, visible=len(uploaded_images) > 0) # return signature is array of gradio components def ui(self, _is_img2img): @@ -95,7 +101,7 @@ class Script(scripts.Script): try: if len(gallery) == 0: from modules.api.api import decode_base64_to_image - images = getattr(p, 'pulid_images', self.images) + images = getattr(p, 'pulid_images', uploaded_images) images = [decode_base64_to_image(image) if isinstance(image, str) else image for image in images] else: images = [Image.open(f['name']) if isinstance(f, dict) else f for f in gallery] @@ -134,6 +140,8 @@ class Script(scripts.Script): ortho = getattr(p, 'pulid_ortho', ortho) sampler = getattr(p, 'pulid_sampler', sampler) sampler_fn = getattr(self.pulid.sampling, f'sample_{sampler}', None) + self.mask_apply_overlay = shared.opts.mask_apply_overlay + shared.opts.data['mask_apply_overlay'] = False if sampler_fn is None: sampler_fn = self.pulid.sampling.sample_dpmpp_2m_sde @@ -149,7 +157,7 @@ class Script(scripts.Script): ) shared.sd_model.no_recurse = True sd_models.copy_diffuser_options(shared.sd_model, shared.sd_model.pipe) - # sd_models.move_model(shared.sd_model, devices.device) # move pipeline to device + sd_models.move_model(shared.sd_model, devices.device) # move pipeline to device sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model') devices.torch_gc() except Exception as e: @@ -204,11 +212,12 @@ class Script(scripts.Script): def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=unused-argument _strength, _zero, _sampler, _ortho, _gallery, cache = args - cache = getattr(p, 'pulid_cache', cache) - if cache: - shared.log.debug(f'PuLID cache: class={shared.sd_model.__class__.__name__}') - return processed if hasattr(shared.sd_model, 'pipe') and shared.sd_model_type == "sdxl": + shared.opts.data['mask_apply_overlay'] = self.mask_apply_overlay + cache = getattr(p, 'pulid_cache', cache) + if cache: + shared.log.debug(f'PuLID cache: class={shared.sd_model.__class__.__name__}') + return processed if hasattr(shared.sd_model, 'app'): shared.sd_model.app = None shared.sd_model.ip_adapter = None diff --git a/scripts/xyz_grid_classes.py b/scripts/xyz_grid_classes.py index 08ea279f4..c3d3554e2 100644 --- a/scripts/xyz_grid_classes.py +++ b/scripts/xyz_grid_classes.py @@ -136,6 +136,7 @@ axis_options = [ AxisOption("[Postprocess] Upscaler", str, apply_upscaler, cost=0.4, choices=lambda: [x.name for x in shared.sd_upscalers][1:]), AxisOption("[Postprocess] Context", str, apply_context, choices=lambda: ["Add with forward", "Remove with forward", "Add with backward", "Remove with backward"]), AxisOption("[Postprocess] Detailer", str, apply_detailer, fmt=format_value_add_label), + AxisOption("[Postprocess] Detailer strength", str, apply_field("detailer_strength")), AxisOption("[HDR] Mode", int, apply_field("hdr_mode")), AxisOption("[HDR] Brightness", float, apply_field("hdr_brightness")), AxisOption("[HDR] Color", float, apply_field("hdr_color")), From 9afc6b186f4e71813be01f6df16419c40f6e68a6 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 8 Nov 2024 14:06:28 -0500 Subject: [PATCH 29/40] update changelog Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b989132f2..417165276 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ This release can be considered an LTS release before we kick off the next round - advanced method of face transfer with better quality as well as control over identity and appearance try it out, likely the best quality available for sdxl models - select in *scripts -> pulid* - - compatible with *sdxl* for text-to-image and image-to-image + - compatible with *sdxl* for text-to-image, image-to-image, inpaint and detailer workflows - can be used in xyz grid - *note*: this module contains several advanced features on top of original implementation - [InstantIR](https://github.com/instantX-research/InstantIR): Blind Image Restoration with Instant Generative Reference From 94e188eab3762bfc64a371e9f6c57a6c27ffb158 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 8 Nov 2024 18:07:59 -0500 Subject: [PATCH 30/40] pulid fix seed Signed-off-by: Vladimir Mandic --- modules/processing_args.py | 2 ++ scripts/pulid_ext.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/processing_args.py b/modules/processing_args.py index 1cb9457c5..5b320f9ea 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -182,6 +182,8 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 if hasattr(model, 'scheduler') and hasattr(model.scheduler, 'noise_sampler_seed') and hasattr(model.scheduler, 'noise_sampler'): model.scheduler.noise_sampler = None # noise needs to be reset instead of using cached values model.scheduler.noise_sampler_seed = p.seeds # some schedulers have internal noise generator and do not use pipeline generator + if 'seed' in possible: + args['seed'] = p.seed if 'noise_sampler_seed' in possible: args['noise_sampler_seed'] = p.seeds if 'guidance_scale' in possible: diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 7ec32e904..d0d738b6a 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -174,10 +174,10 @@ class Script(scripts.Script): shared.sd_model.debug_img_list = [] uncond_id_embedding, id_embedding = shared.sd_model.get_id_embedding(images) + p.seed = processing_helpers.get_fixed_seed(p.seed) if direct: # run pipeline directly shared.state.begin('PuLID') processing.fix_seed(p) - p.seed = processing_helpers.get_fixed_seed(p.seed) p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) with devices.inference_context(): From c199e3e66883f3db1221a54490ea910157c1d4c0 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 9 Nov 2024 20:03:41 -0500 Subject: [PATCH 31/40] pulid optimizations: dtype, vae, offload Signed-off-by: Vladimir Mandic --- modules/processing_diffusers.py | 14 +++-- modules/processing_vae.py | 11 +++- modules/pulid/attention_processor.py | 15 ++--- modules/pulid/eva_clip/pretrained.py | 3 +- modules/pulid/pulid_sdxl.py | 93 +++++++++++++--------------- modules/sd_models.py | 8 ++- scripts/pulid_ext.py | 51 +++++++++------ 7 files changed, 103 insertions(+), 92 deletions(-) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 7ec0dd08a..7537d0209 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -71,7 +71,7 @@ def process_base(p: processing.StableDiffusionProcessing): guidance_rescale=p.diffusers_guidance_rescale, denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None, denoising_end=p.refiner_start if use_refiner_start else 1 if use_denoise_start else None, - output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np', + output_type='latent', clip_skip=p.clip_skip, desc='Base', ) @@ -217,7 +217,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output): eta=shared.opts.scheduler_eta, guidance_scale=p.image_cfg_scale if p.image_cfg_scale is not None else p.cfg_scale, guidance_rescale=p.diffusers_guidance_rescale, - output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np', + output_type='latent', clip_skip=p.clip_skip, image=output.images, strength=p.denoising_strength, @@ -278,7 +278,7 @@ def process_refine(p: processing.StableDiffusionProcessing, output): for i in range(len(output.images)): image = output.images[i] noise_level = round(350 * p.denoising_strength) - output_type='latent' if hasattr(shared.sd_refiner, 'vae') else 'np' + output_type='latent' if 'Upscale' in shared.sd_refiner.__class__.__name__ or 'Flux' in shared.sd_refiner.__class__.__name__: image = processing_vae.vae_decode(latents=image, model=shared.sd_model, full_quality=p.full_quality, output_type='pil', width=p.width, height=p.height) p.extra_generation_params['Noise level'] = noise_level @@ -346,7 +346,11 @@ def process_decode(p: processing.StableDiffusionProcessing, output): if not hasattr(output, 'images') and hasattr(output, 'frames'): shared.log.debug(f'Generated: frames={len(output.frames[0])}') output.images = output.frames[0] - if hasattr(shared.sd_model, "vae") and output.images is not None and len(output.images) > 0: + model = shared.sd_model if not is_refiner_enabled(p) else shared.sd_refiner + if not hasattr(model, 'vae'): + if hasattr(model, 'pipe') and hasattr(model.pipe, 'vae'): + model = model.pipe + if hasattr(model, "vae") and output.images is not None and len(output.images) > 0: if p.hr_resize_mode > 0 and (p.hr_upscaler != 'None' or p.hr_resize_mode == 5): width = max(getattr(p, 'width', 0), getattr(p, 'hr_upscale_to_x', 0)) height = max(getattr(p, 'height', 0), getattr(p, 'hr_upscale_to_y', 0)) @@ -355,7 +359,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output): height = getattr(p, 'height', 0) results = processing_vae.vae_decode( latents = output.images, - model = shared.sd_model if not is_refiner_enabled(p) else shared.sd_refiner, + model = model, full_quality = p.full_quality, width = width, height = height, diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 75347f416..5e6fa68f4 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -35,7 +35,7 @@ def create_latents(image, p, dtype=None, device=None): def full_vae_decode(latents, model): t0 = time.time() - if not hasattr(model, 'vae'): + if model is None or not hasattr(model, 'vae'): shared.log.error('VAE not found in model') return [] if debug: @@ -170,7 +170,14 @@ def vae_decode(latents, model, output_type='np', full_quality=True, width=None, if latents.shape[-1] <= 4: # not a latent, likely an image decoded = latents.float().cpu().numpy() elif full_quality and hasattr(shared.sd_model, "vae"): - decoded = full_vae_decode(latents=latents, model=shared.sd_model) + parent = shared.sd_model if hasattr(shared.sd_model, 'vae') else None + if hasattr(shared.sd_model, 'vae'): + parent = shared.sd_model + elif hasattr(shared.sd_model, 'pipe') and hasattr(shared.sd_model.pipe, 'vae'): + parent = shared.sd_model.pipe + else: + parent = None + decoded = full_vae_decode(latents=latents, model=parent) else: decoded = taesd_vae_decode(latents=latents) diff --git a/modules/pulid/attention_processor.py b/modules/pulid/attention_processor.py index 9756decc1..fa9e4ff82 100644 --- a/modules/pulid/attention_processor.py +++ b/modules/pulid/attention_processor.py @@ -345,10 +345,7 @@ class IDAttnProcessor2_0(torch.nn.Module): value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - hidden_states = F.scaled_dot_product_attention( - query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False - ) - + hidden_states = F.scaled_dot_product_attention(query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False) hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) hidden_states = hidden_states.to(query.dtype) @@ -363,17 +360,15 @@ class IDAttnProcessor2_0(torch.nn.Module): dtype=id_embedding.dtype, device=id_embedding.device, ) - id_key = self.id_to_k(torch.cat((id_embedding, zero_tensor), dim=1)).to(query.dtype) - id_value = self.id_to_v(torch.cat((id_embedding, zero_tensor), dim=1)).to(query.dtype) + id_cat = torch.cat((id_embedding, zero_tensor), dim=1) + id_key = self.id_to_k(id_cat).to(query.dtype) + id_value = self.id_to_v(id_cat).to(query.dtype) id_key = id_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) id_value = id_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - id_hidden_states = F.scaled_dot_product_attention( - query, id_key, id_value, attn_mask=None, dropout_p=0.0, is_causal=False - ) - + id_hidden_states = F.scaled_dot_product_attention(query, id_key, id_value, attn_mask=None, dropout_p=0.0, is_causal=False) id_hidden_states = id_hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) id_hidden_states = id_hidden_states.to(query.dtype) diff --git a/modules/pulid/eva_clip/pretrained.py b/modules/pulid/eva_clip/pretrained.py index a1e55dcf3..bb87c540c 100644 --- a/modules/pulid/eva_clip/pretrained.py +++ b/modules/pulid/eva_clip/pretrained.py @@ -2,7 +2,6 @@ import hashlib import os import urllib import warnings -from functools import partial from typing import Dict, Union from tqdm import tqdm @@ -277,7 +276,7 @@ def download_pretrained_from_url( loop.update(len(buffer)) if expected_sha256 and not hashlib.sha256(open(download_target, "rb").read()).hexdigest().startswith(expected_sha256): - raise RuntimeError(f"Model has been downloaded but the SHA256 checksum does not not match") + raise RuntimeError("Model has been downloaded but the SHA256 checksum does not not match") return download_target diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index fade7509d..8bd28dbe2 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -5,6 +5,7 @@ import numpy as np import torch import torch.nn as nn from diffusers import DPMSolverMultistepScheduler, StableDiffusionXLPipeline +from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput from huggingface_hub import hf_hub_download, snapshot_download from safetensors.torch import load_file @@ -24,14 +25,17 @@ from attention_processor import IDAttnProcessor2_0 as IDAttnProcessor class StableDiffusionXLPuLIDPipeline: - def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, sampler=None, cache_dir=None): + def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, dtype: torch.dtype=None, providers: list=None, offload: bool=True, sampler=None, cache_dir=None): super().__init__() self.device = device + self.dtype = dtype or torch.float16 self.pipe = pipe self.cache_dir = cache_dir + self.offload = offload self.hack_unet_attn_layers(self.pipe.unet) self.pipe.scheduler = DPMSolverMultistepScheduler.from_config(self.pipe.scheduler.config) - self.id_adapter = IDFormer().to(self.device) + self.id_adapter = IDFormer().to(self.device, self.dtype) + self.providers = providers or ['CUDAExecutionProvider', 'CPUExecutionProvider'] # preprocessors # face align and parsing @@ -43,13 +47,12 @@ class StableDiffusionXLPuLIDPipeline: save_ext='png', device=self.device, ) - self.face_helper.face_parse = None self.face_helper.face_parse = init_parsing_model(model_name='bisenet', device=self.device) # clip-vit backbone - model, _, _ = create_model_and_transforms('EVA02-CLIP-L-14-336', 'eva_clip', force_custom_clip=True) - model = model.visual - self.clip_vision_model = model.to(self.device) + eva_precision = 'fp16' if self.dtype == torch.float16 or self.dtype == torch.bfloat16 else 'fp32' + eva_model, _, _ = create_model_and_transforms('EVA02-CLIP-L-14-336', 'eva_clip', force_custom_clip=True, precision=eva_precision, device=self.device) + self.clip_vision_model = eva_model.visual.to(dtype=self.dtype) eva_transform_mean = getattr(self.clip_vision_model, 'image_mean', OPENAI_DATASET_MEAN) eva_transform_std = getattr(self.clip_vision_model, 'image_std', OPENAI_DATASET_STD) if not isinstance(eva_transform_mean, (list, tuple)): @@ -60,13 +63,12 @@ class StableDiffusionXLPuLIDPipeline: self.eva_transform_std = eva_transform_std # antelopev2 - # snapshot_download('DIAMONIK7777/antelopev2', local_dir='models/antelopev2') local_dir = os.path.join(self.cache_dir, 'pulid', 'models', 'antelopev2') _loc = snapshot_download('DIAMONIK7777/antelopev2', local_dir=local_dir) self.app = FaceAnalysis( name='antelopev2', root=os.path.join(self.cache_dir, 'pulid'), - providers=['CUDAExecutionProvider', 'CPUExecutionProvider'], + providers=self.providers, ) self.app.prepare(ctx_id=0, det_size=(640, 640)) self.handler_ante = insightface.model_zoo.get_model(os.path.join(local_dir, 'glintr100.onnx')) @@ -89,8 +91,12 @@ class StableDiffusionXLPuLIDPipeline: self.log_sigmas = self.sigmas.log() self.sigma_data = 1.0 + # default scheduler if sampler is not None: self.sampler = sampler + else: + from modules.pulid import sampling + self.sampler = sampling.sample_dpmpp_sde @property def sigma_min(self): @@ -130,7 +136,7 @@ class StableDiffusionXLPuLIDPipeline: id_adapter_attn_procs[name] = IDAttnProcessor( hidden_size=hidden_size, cross_attention_dim=cross_attention_dim, - ).to(unet.device) + ).to(unet.device, unet.dtype) else: id_adapter_attn_procs[name] = AttnProcessor() unet.set_attn_processor(id_adapter_attn_procs) @@ -144,7 +150,7 @@ class StableDiffusionXLPuLIDPipeline: module = k.split('.')[0] state_dict_dict.setdefault(module, {}) new_k = k[len(module) + 1 :] - state_dict_dict[module][new_k] = v + state_dict_dict[module][new_k] = v.to(self.dtype) for module in state_dict_dict: getattr(self, module).load_state_dict(state_dict_dict[module], strict=True) @@ -161,24 +167,17 @@ class StableDiffusionXLPuLIDPipeline: """ id_cond_list = [] id_vit_hidden_list = [] + self.face_helper.face_det.to(self.device) + self.clip_vision_model.to(self.device) for _ii, image in enumerate(image_list): self.face_helper.clean_all() image_bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # get antelopev2 embedding face_info = self.app.get(image_bgr) if len(face_info) > 0: - face_info = sorted( - face_info, key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]) - )[ - -1 - ] # only use the maximum face + face_info = sorted(face_info, key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[-1] # only use the maximum face id_ante_embedding = face_info['embedding'] - self.debug_img_list.append( - image[ - int(face_info['bbox'][1]) : int(face_info['bbox'][3]), - int(face_info['bbox'][0]) : int(face_info['bbox'][2]), - ] - ) + self.debug_img_list.append(image[int(face_info['bbox'][1]) : int(face_info['bbox'][3]), int(face_info['bbox'][0]) : int(face_info['bbox'][2])]) else: id_ante_embedding = None @@ -210,13 +209,9 @@ class StableDiffusionXLPuLIDPipeline: self.debug_img_list.append(tensor2img(face_features_image, rgb2bgr=False)) # transform img before sending to eva-clip-vit - face_features_image = resize( - face_features_image, self.clip_vision_model.image_size, InterpolationMode.BICUBIC - ) - face_features_image = normalize(face_features_image, self.eva_transform_mean, self.eva_transform_std) - id_cond_vit, id_vit_hidden = self.clip_vision_model( - face_features_image, return_all_features=False, return_hidden=True, shuffle=False - ) + face_features_image = resize(face_features_image, self.clip_vision_model.image_size, InterpolationMode.BICUBIC) + face_features_image = normalize(face_features_image, self.eva_transform_mean, self.eva_transform_std).to(self.dtype) + id_cond_vit, id_vit_hidden = self.clip_vision_model(face_features_image, return_all_features=False, return_hidden=True, shuffle=False) id_cond_vit_norm = torch.norm(id_cond_vit, 2, 1, True) id_cond_vit = torch.div(id_cond_vit, id_cond_vit_norm) @@ -225,19 +220,25 @@ class StableDiffusionXLPuLIDPipeline: id_cond_list.append(id_cond) id_vit_hidden_list.append(id_vit_hidden) - id_uncond = torch.zeros_like(id_cond_list[0]) + self.id_adapter.to(self.device) + id_uncond = torch.zeros_like(id_cond_list[0]).to(self.dtype) id_vit_hidden_uncond = [] for layer_idx in range(0, len(id_vit_hidden_list[0])): - id_vit_hidden_uncond.append(torch.zeros_like(id_vit_hidden_list[0][layer_idx])) + id_vit_hidden_uncond.append(torch.zeros_like(id_vit_hidden_list[0][layer_idx]).to(self.dtype)) - id_cond = torch.stack(id_cond_list, dim=1) + id_cond = torch.stack(id_cond_list, dim=1).to(self.dtype) id_vit_hidden = id_vit_hidden_list[0] for i in range(1, len(image_list)): for j, x in enumerate(id_vit_hidden_list[i]): - id_vit_hidden[j] = torch.cat([id_vit_hidden[j], x], dim=1) + id_vit_hidden[j] = torch.cat([id_vit_hidden[j], x], dim=1).to(self.dtype) id_embedding = self.id_adapter(id_cond, id_vit_hidden) uncond_id_embedding = self.id_adapter(id_uncond, id_vit_hidden_uncond) + if self.offload: + self.face_helper.face_det.to('cpu') + self.id_adapter.to('cpu') + self.clip_vision_model.to('cpu') + # return id_embedding return uncond_id_embedding, id_embedding @@ -314,6 +315,7 @@ class StableDiffusionXLPuLIDPipeline: id_embedding=None, uncond_id_embedding=None, id_scale: float=1.0, + output_type: str='pil', callback_on_step_end=None, ): self.step = 0 # pylint: disable=attribute-defined-outside-init @@ -370,24 +372,15 @@ class StableDiffusionXLPuLIDPipeline: mask_args = None latents = self.sampler(self.sample, noisy_latent, sigmas, extra_args=sampler_kwargs, disable=False, mask_args=mask_args) - latents = latents.to(dtype=self.pipe.vae.dtype, device=self.device) / self.pipe.vae.config.scaling_factor - images = self.pipe.vae.decode(latents).sample - images = self.pipe.image_processor.postprocess(images, output_type='pil') - - # Pixel space final mask - # if mask_image is not None: - # # TODO: Fix XYZ - # from PIL import Image - # mask_image = np.asarray(mask_image.convert("L")) - # mask_image = mask_image / mask_image.max() - # mask_image = mask_image.reshape(1,mask_image.shape[0],mask_image.shape[1],1) - # image = np.asarray(image).astype(mask_image.dtype) - # images = np.asarray(images).astype(mask_image.dtype) - # images = ((1 - mask_image) * image) + (mask_image * images) - # images = images[0].round().astype(np.uint8) - # images = [Image.fromarray(images)] - - return images + if output_type == 'latent': + images = self.pipe.image_processor.postprocess(latents, output_type='latent') + elif output_type == 'np': + images = self.pipe.image_processor.postprocess(latents, output_type='np') + else: + latents = latents.to(dtype=self.pipe.vae.dtype, device=self.device) / self.pipe.vae.config.scaling_factor + images = self.pipe.vae.decode(latents).sample + images = self.pipe.image_processor.postprocess(images, output_type='pil') + return StableDiffusionXLPipelineOutput(images) class StableDiffusionXLPuLIDPipelineImage(StableDiffusionXLPuLIDPipeline): diff --git a/modules/sd_models.py b/modules/sd_models.py index 8320ac293..e139895ea 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -402,9 +402,11 @@ def apply_balanced_offload(sd_model): def apply_balanced_offload_to_module(pipe): if hasattr(pipe, "pipe"): apply_balanced_offload_to_module(pipe.pipe) - if not hasattr(pipe, "_internal_dict"): - return - for module_name in pipe._internal_dict.keys(): # pylint: disable=protected-access + if hasattr(pipe, "_internal_dict"): + keys = pipe._internal_dict.keys() # pylint: disable=protected-access + else: + keys = get_signature(shared.sd_model).keys() + for module_name in keys: # pylint: disable=protected-access module = getattr(pipe, module_name, None) if isinstance(module, torch.nn.Module): checkpoint_name = pipe.sd_checkpoint_info.name if getattr(pipe, "sd_checkpoint_info", None) is not None else None diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index d0d738b6a..6599fa2e8 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -1,5 +1,6 @@ import io import os +import time import contextlib import gradio as gr import numpy as np @@ -18,6 +19,7 @@ class Script(scripts.Script): self.pulid = None self.cache = None self.mask_apply_overlay = shared.opts.mask_apply_overlay + self.preprocess = 0 super().__init__() self.register() # pulid is script with processing override so xyz doesnt execute @@ -88,15 +90,16 @@ class Script(scripts.Script): sampler = gr.Dropdown(label="Sampler", value='dpmpp_sde', choices=['dpmpp_2m', 'dpmpp_2m_sde', 'dpmpp_2s_ancestral', 'dpmpp_3m_sde', 'dpmpp_sde', 'euler', 'euler_ancestral']) ortho = gr.Dropdown(label="Ortho", choices=['off', 'v1', 'v2'], value='v2') with gr.Row(): - cache = gr.Checkbox(label='Keep model', value=False) + restore = gr.Checkbox(label='Restore pipe on end', value=False) + offload = gr.Checkbox(label='Offload face module', value=True) with gr.Row(): files = gr.File(label='Input images', file_count='multiple', file_types=['image'], type='file', interactive=True, height=100) with gr.Row(): gallery = gr.Gallery(show_label=False, value=[], visible=False, container=False, rows=1) files.change(fn=self.load_images, inputs=[files], outputs=[gallery]) - return [strength, zero, sampler, ortho, gallery, cache] + return [strength, zero, sampler, ortho, gallery, restore, offload] - def run(self, p: processing.StableDiffusionProcessing, strength: float = 0.8, zero: int = 20, sampler: str = 'dpmpp_sde', ortho: str = 'v2', gallery: list = [], cache: bool = False): # pylint: disable=arguments-differ, unused-argument + def run(self, p: processing.StableDiffusionProcessing, strength: float = 0.8, zero: int = 20, sampler: str = 'dpmpp_sde', ortho: str = 'v2', gallery: list = [], restore: bool = False, offload: bool = True): # pylint: disable=arguments-differ, unused-argument images = [] try: if len(gallery) == 0: @@ -135,13 +138,13 @@ class Script(scripts.Script): shared.log.warning('PuLID: batch size not supported') p.batch_size = 1 + self.mask_apply_overlay = shared.opts.mask_apply_overlay + shared.opts.data['mask_apply_overlay'] = False strength = getattr(p, 'pulid_strength', strength) zero = getattr(p, 'pulid_zero', zero) ortho = getattr(p, 'pulid_ortho', ortho) sampler = getattr(p, 'pulid_sampler', sampler) sampler_fn = getattr(self.pulid.sampling, f'sample_{sampler}', None) - self.mask_apply_overlay = shared.opts.mask_apply_overlay - shared.opts.data['mask_apply_overlay'] = False if sampler_fn is None: sampler_fn = self.pulid.sampling.sample_dpmpp_2m_sde @@ -153,6 +156,9 @@ class Script(scripts.Script): shared.sd_model = self.pulid.StableDiffusionXLPuLIDPipeline( pipe =shared.sd_model, device=devices.device, + dtype=devices.dtype, + providers=devices.onnx, + offload=offload, cache_dir=shared.opts.hfcache_dir, ) shared.sd_model.no_recurse = True @@ -166,13 +172,20 @@ class Script(scripts.Script): return None shared.sd_model.sampler = sampler_fn - shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} strength={strength} zero={zero} ortho={ortho} sampler={sampler_fn} images={[i.shape for i in images]}') + shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} strength={strength} zero={zero} ortho={ortho} sampler={sampler_fn} images={[i.shape for i in images]} offload={offload}') self.pulid.attention.NUM_ZERO = zero self.pulid.attention.ORTHO = ortho == 'v1' self.pulid.attention.ORTHO_v2 = ortho == 'v2' images = [self.pulid.resize(image, 1024) for image in images] shared.sd_model.debug_img_list = [] + + # get id embedding used for attention + t0 = time.time() uncond_id_embedding, id_embedding = shared.sd_model.get_id_embedding(images) + if offload: + devices.torch_gc() + t1 = time.time() + self.preprocess = t1-t0 p.seed = processing_helpers.get_fixed_seed(p.seed) if direct: # run pipeline directly @@ -211,20 +224,18 @@ class Script(scripts.Script): return processed def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=unused-argument - _strength, _zero, _sampler, _ortho, _gallery, cache = args + _strength, _zero, _sampler, _ortho, _gallery, restore, _offload = args if hasattr(shared.sd_model, 'pipe') and shared.sd_model_type == "sdxl": shared.opts.data['mask_apply_overlay'] = self.mask_apply_overlay - cache = getattr(p, 'pulid_cache', cache) - if cache: - shared.log.debug(f'PuLID cache: class={shared.sd_model.__class__.__name__}') - return processed - if hasattr(shared.sd_model, 'app'): - shared.sd_model.app = None - shared.sd_model.ip_adapter = None - shared.sd_model.face_helper = None - shared.sd_model.clip_vision_model = None - shared.sd_model.handler_ante = None - shared.sd_model = shared.sd_model.pipe - devices.torch_gc(force=True) - shared.log.debug(f'PuLID restore: class={shared.sd_model.__class__.__name__}') + restore = getattr(p, 'pulid_restore', restore) + if restore: + if hasattr(shared.sd_model, 'app'): + shared.sd_model.app = None + shared.sd_model.ip_adapter = None + shared.sd_model.face_helper = None + shared.sd_model.clip_vision_model = None + shared.sd_model.handler_ante = None + shared.sd_model = shared.sd_model.pipe + devices.torch_gc(force=True) + shared.log.debug(f'PuLID complete: class={shared.sd_model.__class__.__name__} preprocess={self.preprocess:.2f} pipe={"restore" if restore else "cache"}') return processed From 1c74d36e29f033117f1b5d867c9ff0a07232ffa2 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 10 Nov 2024 09:42:05 -0500 Subject: [PATCH 32/40] pullid support sdpa add both v1.0 and v1.1 models Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 +- modules/errors.py | 4 +- modules/pulid/encoders_transformer.py | 68 ++++++++++++++++++++++++--- modules/pulid/pulid_sdxl.py | 38 +++++++++++---- scripts/pulid_ext.py | 25 ++++++++-- 5 files changed, 113 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 417165276..9affd53db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-11-08 +## Update for 2024-11-10 Smaller release just few days after the last one, but with some important fixes and improvements. This release can be considered an LTS release before we kick off the next round of major updates. diff --git a/modules/errors.py b/modules/errors.py index c4d66c351..527884cf1 100644 --- a/modules/errors.py +++ b/modules/errors.py @@ -59,7 +59,7 @@ def exception(suppress=[]): console.print_exception(show_locals=False, max_frames=16, extra_lines=2, suppress=suppress, theme="ansi_dark", word_wrap=False, width=min([console.width, 200])) -def profile(profiler, msg: str): +def profile(profiler, msg: str, n: int = 5): profiler.disable() import io import pstats @@ -83,7 +83,7 @@ def profile(profiler, msg: str): and 'rich' not in x and x.strip() != '' ] - txt = '\n'.join(lines[:min(5, len(lines))]) + txt = '\n'.join(lines[:min(n, len(lines))]) log.debug(f'Profile {msg}: {txt}') diff --git a/modules/pulid/encoders_transformer.py b/modules/pulid/encoders_transformer.py index d1ecef2c6..ae245044b 100644 --- a/modules/pulid/encoders_transformer.py +++ b/modules/pulid/encoders_transformer.py @@ -186,23 +186,79 @@ class IDFormer(nn.Module): ) def forward(self, x, y): - latents = self.latents.repeat(x.size(0), 1, 1) - num_duotu = x.shape[1] if x.ndim == 3 else 1 - x = self.id_embedding_mapping(x) x = x.reshape(-1, self.num_id_token * num_duotu, self.dim) - latents = torch.cat((latents, x), dim=1) - for i in range(5): vit_feature = getattr(self, f'mapping_{i}')(y[i]) ctx_feature = torch.cat((x, vit_feature), dim=1) for attn, ff in self.layers[i * self.depth: (i + 1) * self.depth]: latents = attn(ctx_feature, latents) + latents latents = ff(latents) + latents - latents = latents[:, :self.num_queries] latents = latents @ self.proj_out return latents + + +class IDEncoder(nn.Module): + def __init__(self, width=1280, context_dim=2048, num_token=5): + super().__init__() + self.num_token = num_token + self.context_dim = context_dim + h1 = min((context_dim * num_token) // 4, 1024) + h2 = min((context_dim * num_token) // 2, 1024) + self.body = nn.Sequential( + nn.Linear(width, h1), + nn.LayerNorm(h1), + nn.LeakyReLU(), + nn.Linear(h1, h2), + nn.LayerNorm(h2), + nn.LeakyReLU(), + nn.Linear(h2, context_dim * num_token), + ) + + for i in range(5): + setattr( + self, + f'mapping_{i}', + nn.Sequential( + nn.Linear(1024, 1024), + nn.LayerNorm(1024), + nn.LeakyReLU(), + nn.Linear(1024, 1024), + nn.LayerNorm(1024), + nn.LeakyReLU(), + nn.Linear(1024, context_dim), + ), + ) + + setattr( + self, + f'mapping_patch_{i}', + nn.Sequential( + nn.Linear(1024, 1024), + nn.LayerNorm(1024), + nn.LeakyReLU(), + nn.Linear(1024, 1024), + nn.LayerNorm(1024), + nn.LeakyReLU(), + nn.Linear(1024, context_dim), + ), + ) + + def forward(self, x, y): + # x shape [N, C] + x = self.body(x) + x = x.reshape(-1, self.num_token, self.context_dim) + + hidden_states = () + for i, emb in enumerate(y): + hidden_state = getattr(self, f'mapping_{i}')(emb[:, :1]) + getattr(self, f'mapping_patch_{i}')( + emb[:, 1:] + ).mean(dim=1, keepdim=True) + hidden_states += (hidden_state,) + hidden_states = torch.cat(hidden_states, dim=1) + + return torch.cat([x, hidden_states], dim=1) diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index 8bd28dbe2..d2a761654 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -19,23 +19,28 @@ from insightface.app import FaceAnalysis from eva_clip import create_model_and_transforms from eva_clip.constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD -from encoders_transformer import IDFormer -from attention_processor import AttnProcessor2_0 as AttnProcessor -from attention_processor import IDAttnProcessor2_0 as IDAttnProcessor +from encoders_transformer import IDFormer, IDEncoder class StableDiffusionXLPuLIDPipeline: - def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, dtype: torch.dtype=None, providers: list=None, offload: bool=True, sampler=None, cache_dir=None): + def __init__(self, pipe: StableDiffusionXLPipeline, device: torch.device, dtype: torch.dtype=None, providers: list=None, offload: bool=True, sampler=None, cache_dir=None, sdp: bool=True, version: str='v1.1'): super().__init__() self.device = device self.dtype = dtype or torch.float16 self.pipe = pipe self.cache_dir = cache_dir self.offload = offload - self.hack_unet_attn_layers(self.pipe.unet) + self.sdp = sdp + self.version = version + self.folder = 'models--ToTheBeginning--PuLID' + self.pipe.scheduler = DPMSolverMultistepScheduler.from_config(self.pipe.scheduler.config) - self.id_adapter = IDFormer().to(self.device, self.dtype) + if self.version == 'v1.1': + self.id_adapter = IDFormer().to(self.device, self.dtype) + else: + self.id_adapter = IDEncoder().to(self.device, self.dtype) self.providers = providers or ['CUDAExecutionProvider', 'CPUExecutionProvider'] + self.hack_unet_attn_layers(self.pipe.unet) # preprocessors # face align and parsing @@ -63,11 +68,11 @@ class StableDiffusionXLPuLIDPipeline: self.eva_transform_std = eva_transform_std # antelopev2 - local_dir = os.path.join(self.cache_dir, 'pulid', 'models', 'antelopev2') + local_dir = os.path.join(self.cache_dir, self.folder, 'models', 'antelopev2') _loc = snapshot_download('DIAMONIK7777/antelopev2', local_dir=local_dir) self.app = FaceAnalysis( name='antelopev2', - root=os.path.join(self.cache_dir, 'pulid'), + root=os.path.join(self.cache_dir, self.folder), providers=self.providers, ) self.app.prepare(ctx_id=0, det_size=(640, 640)) @@ -119,6 +124,12 @@ class StableDiffusionXLPuLIDPipeline: return torch.cat([sigmas, sigmas.new_zeros([1])]) def hack_unet_attn_layers(self, unet): + if self.sdp: + from attention_processor import AttnProcessor2_0 as AttnProcessor + from attention_processor import IDAttnProcessor2_0 as IDAttnProcessor + else: + from attention_processor import AttnProcessor + from attention_processor import IDAttnProcessor id_adapter_attn_procs = {} for name, _ in unet.attn_processors.items(): cross_attention_dim = None if name.endswith("attn1.processor") else unet.config.cross_attention_dim @@ -143,8 +154,12 @@ class StableDiffusionXLPuLIDPipeline: self.id_adapter_attn_layers = nn.ModuleList(unet.attn_processors.values()) def load_pretrain(self): - ckpt_path = hf_hub_download('guozinan/PuLID', 'pulid_v1.1.safetensors', local_dir=os.path.join(self.cache_dir, 'pulid')) - state_dict = load_file(ckpt_path) + if self.version == 'v1.1': + ckpt_path = hf_hub_download('guozinan/PuLID', 'pulid_v1.1.safetensors', local_dir=os.path.join(self.cache_dir, self.folder)) + state_dict = load_file(ckpt_path) + else: + ckpt_path = hf_hub_download('guozinan/PuLID', 'pulid_v1.bin', local_dir=os.path.join(self.cache_dir, self.folder)) + state_dict = torch.load(ckpt_path, map_location="cpu") state_dict_dict = {} for k, v in state_dict.items(): module = k.split('.')[0] @@ -371,7 +386,10 @@ class StableDiffusionXLPuLIDPipeline: else: mask_args = None + # actual sampling loop latents = self.sampler(self.sample, noisy_latent, sigmas, extra_args=sampler_kwargs, disable=False, mask_args=mask_args) + + # process output if output_type == 'latent': images = self.pipe.image_processor.postprocess(latents, output_type='latent') elif output_type == 'np': diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 6599fa2e8..181e954db 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -89,6 +89,8 @@ class Script(scripts.Script): with gr.Row(): sampler = gr.Dropdown(label="Sampler", value='dpmpp_sde', choices=['dpmpp_2m', 'dpmpp_2m_sde', 'dpmpp_2s_ancestral', 'dpmpp_3m_sde', 'dpmpp_sde', 'euler', 'euler_ancestral']) ortho = gr.Dropdown(label="Ortho", choices=['off', 'v1', 'v2'], value='v2') + with gr.Row(): + version = gr.Dropdown(label="Version", value='v1.1', choices=['v1.0', 'v1.1']) with gr.Row(): restore = gr.Checkbox(label='Restore pipe on end', value=False) offload = gr.Checkbox(label='Offload face module', value=True) @@ -97,9 +99,20 @@ class Script(scripts.Script): with gr.Row(): gallery = gr.Gallery(show_label=False, value=[], visible=False, container=False, rows=1) files.change(fn=self.load_images, inputs=[files], outputs=[gallery]) - return [strength, zero, sampler, ortho, gallery, restore, offload] + return [strength, zero, sampler, ortho, gallery, restore, offload, version] - def run(self, p: processing.StableDiffusionProcessing, strength: float = 0.8, zero: int = 20, sampler: str = 'dpmpp_sde', ortho: str = 'v2', gallery: list = [], restore: bool = False, offload: bool = True): # pylint: disable=arguments-differ, unused-argument + def run( + self, + p: processing.StableDiffusionProcessing, + strength: float = 0.8, + zero: int = 20, + sampler: str = 'dpmpp_sde', + ortho: str = 'v2', + gallery: list = [], + restore: bool = False, + offload: bool = True, + version: str = 'v1.1' + ): # pylint: disable=arguments-differ, unused-argument images = [] try: if len(gallery) == 0: @@ -154,11 +167,13 @@ class Script(scripts.Script): ctx = contextlib.nullcontext() if debug else contextlib.redirect_stdout(stdout) with ctx: shared.sd_model = self.pulid.StableDiffusionXLPuLIDPipeline( - pipe =shared.sd_model, + pipe=shared.sd_model, device=devices.device, dtype=devices.dtype, providers=devices.onnx, offload=offload, + version=version, + sdp=shared.opts.cross_attention_optimization == "Scaled-Dot-Product", cache_dir=shared.opts.hfcache_dir, ) shared.sd_model.no_recurse = True @@ -172,7 +187,7 @@ class Script(scripts.Script): return None shared.sd_model.sampler = sampler_fn - shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} strength={strength} zero={zero} ortho={ortho} sampler={sampler_fn} images={[i.shape for i in images]} offload={offload}') + shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} version="{version}" strength={strength} zero={zero} ortho={ortho} sampler={sampler_fn} images={[i.shape for i in images]} offload={offload}') self.pulid.attention.NUM_ZERO = zero self.pulid.attention.ORTHO = ortho == 'v1' self.pulid.attention.ORTHO_v2 = ortho == 'v2' @@ -224,7 +239,7 @@ class Script(scripts.Script): return processed def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=unused-argument - _strength, _zero, _sampler, _ortho, _gallery, restore, _offload = args + _strength, _zero, _sampler, _ortho, _gallery, restore, _offload, _version = args if hasattr(shared.sd_model, 'pipe') and shared.sd_model_type == "sdxl": shared.opts.data['mask_apply_overlay'] = self.mask_apply_overlay restore = getattr(p, 'pulid_restore', restore) From 3cd21d6b74f83304fce67b2a8704eaf69b7b4acd Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 10 Nov 2024 16:36:08 -0500 Subject: [PATCH 33/40] css fix margin Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 5 +++-- javascript/sdnext.css | 3 ++- modules/pulid/encoders_transformer.py | 14 -------------- scripts/pulid_ext.py | 5 +++-- 4 files changed, 8 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9affd53db..5db77983c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ This release can be considered an LTS release before we kick off the next round - major [Wiki](https://github.com/vladmandic/automatic/wiki) and [Home](https://github.com/vladmandic/automatic) updates - Integrations: - [PuLID](https://github.com/ToTheBeginning/PuLID): Pure and Lightning ID Customization via Contrastive Alignment - - advanced method of face transfer with better quality as well as control over identity and appearance + - advanced method of face id transfer with better quality as well as control over identity and appearance try it out, likely the best quality available for sdxl models - select in *scripts -> pulid* - compatible with *sdxl* for text-to-image, image-to-image, inpaint and detailer workflows @@ -98,7 +98,8 @@ This release can be considered an LTS release before we kick off the next round - fix network height in standard vs modern ui - fix k-diff enum on startup - fix text2video scripts - - dont uninstall flash-attn + - dont uninstall flash-attn + - ui css fixes - move downloads of some auxillary models to hfcache instead of models folder ## Update for 2024-10-29 diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 192d0d487..08fae2eb8 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -13,11 +13,12 @@ footer { display: none; margin-top: 0 !important;} table { overflow-x: auto !important; overflow-y: auto !important; } td { border-bottom: none !important; padding: 0 0.5em !important; } tr { border-bottom: none !important; padding: 0 0.5em !important; } +td > div > span { overflow-y: auto; max-height: 3em; overflow-x: hidden; } textarea { overflow-y: auto !important; } span { font-size: var(--text-md) !important; } button { font-size: var(--text-lg) !important; } input[type='color'] { width: 64px; height: 32px; } -td > div > span { overflow-y: auto; max-height: 3em; overflow-x: hidden; } +input::-webkit-outer-spin-button, input::-webkit-inner-spin-button { margin-left: 4px; } /* gradio elements */ .block .padded:not(.gradio-accordion) { padding: 4px 0 0 0 !important; margin-right: 0; min-width: 90px !important; } diff --git a/modules/pulid/encoders_transformer.py b/modules/pulid/encoders_transformer.py index ae245044b..834d5aa94 100644 --- a/modules/pulid/encoders_transformer.py +++ b/modules/pulid/encoders_transformer.py @@ -32,10 +32,8 @@ class PerceiverAttentionCA(nn.Module): self.dim_head = dim_head self.heads = heads inner_dim = dim_head * heads - self.norm1 = nn.LayerNorm(dim if kv_dim is None else kv_dim) self.norm2 = nn.LayerNorm(dim) - self.to_q = nn.Linear(dim, inner_dim, bias=False) self.to_kv = nn.Linear(dim if kv_dim is None else kv_dim, inner_dim * 2, bias=False) self.to_out = nn.Linear(inner_dim, dim, bias=False) @@ -50,12 +48,9 @@ class PerceiverAttentionCA(nn.Module): """ x = self.norm1(x) latents = self.norm2(latents) - b, seq_len, _ = latents.shape - q = self.to_q(latents) k, v = self.to_kv(x).chunk(2, dim=-1) - q = reshape_tensor(q, self.heads) k = reshape_tensor(k, self.heads) v = reshape_tensor(v, self.heads) @@ -65,7 +60,6 @@ class PerceiverAttentionCA(nn.Module): weight = (q * scale) @ (k * scale).transpose(-2, -1) # More stable with f16 than dividing afterwards weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype) out = weight @ v - out = out.permute(0, 2, 1, 3).reshape(b, seq_len, -1) return self.to_out(out) @@ -78,10 +72,8 @@ class PerceiverAttention(nn.Module): self.dim_head = dim_head self.heads = heads inner_dim = dim_head * heads - self.norm1 = nn.LayerNorm(dim if kv_dim is None else kv_dim) self.norm2 = nn.LayerNorm(dim) - self.to_q = nn.Linear(dim, inner_dim, bias=False) self.to_kv = nn.Linear(dim if kv_dim is None else kv_dim, inner_dim * 2, bias=False) self.to_out = nn.Linear(inner_dim, dim, bias=False) @@ -96,13 +88,10 @@ class PerceiverAttention(nn.Module): """ x = self.norm1(x) latents = self.norm2(latents) - b, seq_len, _ = latents.shape - q = self.to_q(latents) kv_input = torch.cat((x, latents), dim=-2) k, v = self.to_kv(kv_input).chunk(2, dim=-1) - q = reshape_tensor(q, self.heads) k = reshape_tensor(k, self.heads) v = reshape_tensor(v, self.heads) @@ -112,7 +101,6 @@ class PerceiverAttention(nn.Module): weight = (q * scale) @ (k * scale).transpose(-2, -1) # More stable with f16 than dividing afterwards weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype) out = weight @ v - out = out.permute(0, 2, 1, 3).reshape(b, seq_len, -1) return self.to_out(out) @@ -145,7 +133,6 @@ class IDFormer(nn.Module): assert depth % 5 == 0 self.depth = depth // 5 scale = dim ** -0.5 - self.latents = nn.Parameter(torch.randn(1, num_queries, dim) * scale) self.proj_out = nn.Parameter(scale * torch.randn(dim, output_dim)) @@ -233,7 +220,6 @@ class IDEncoder(nn.Module): nn.Linear(1024, context_dim), ), ) - setattr( self, f'mapping_patch_{i}', diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 181e954db..43039d73a 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -153,6 +153,7 @@ class Script(scripts.Script): self.mask_apply_overlay = shared.opts.mask_apply_overlay shared.opts.data['mask_apply_overlay'] = False + sdp = shared.opts.cross_attention_optimization == "Scaled-Dot-Product" strength = getattr(p, 'pulid_strength', strength) zero = getattr(p, 'pulid_zero', zero) ortho = getattr(p, 'pulid_ortho', ortho) @@ -173,7 +174,7 @@ class Script(scripts.Script): providers=devices.onnx, offload=offload, version=version, - sdp=shared.opts.cross_attention_optimization == "Scaled-Dot-Product", + sdp=sdp, cache_dir=shared.opts.hfcache_dir, ) shared.sd_model.no_recurse = True @@ -187,7 +188,7 @@ class Script(scripts.Script): return None shared.sd_model.sampler = sampler_fn - shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} version="{version}" strength={strength} zero={zero} ortho={ortho} sampler={sampler_fn} images={[i.shape for i in images]} offload={offload}') + shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} version="{version}" sdp={sdp} strength={strength} zero={zero} ortho={ortho} sampler={sampler_fn} images={[i.shape for i in images]} offload={offload}') self.pulid.attention.NUM_ZERO = zero self.pulid.attention.ORTHO = ortho == 'v1' self.pulid.attention.ORTHO_v2 = ortho == 'v2' From 15381eb1038b010fe32b3acfc756a76ff317c49f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 11 Nov 2024 09:53:05 -0500 Subject: [PATCH 34/40] add pulid debug Signed-off-by: Vladimir Mandic --- installer.py | 3 +++ modules/pulid/pulid_sdxl.py | 38 +++++++++++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/installer.py b/installer.py index b19eae87e..e50271280 100644 --- a/installer.py +++ b/installer.py @@ -110,6 +110,9 @@ def setup_logging(): "traceback.border": "black", "traceback.border.syntax_error": "black", "inspect.value.border": "black", + "logging.level.info": "blue_violet", + "logging.level.debug": "purple4", + "logging.level.trace": "dark_blue", })) logging.basicConfig(level=logging.ERROR, format='%(asctime)s | %(name)s | %(levelname)s | %(module)s | %(message)s', handlers=[logging.NullHandler()]) # redirect default logger to null pretty_install(console=console) diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index d2a761654..3053d759d 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -4,7 +4,7 @@ import insightface import numpy as np import torch import torch.nn as nn -from diffusers import DPMSolverMultistepScheduler, StableDiffusionXLPipeline +from diffusers import StableDiffusionXLPipeline from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput from huggingface_hub import hf_hub_download, snapshot_download @@ -20,6 +20,10 @@ from insightface.app import FaceAnalysis from eva_clip import create_model_and_transforms from eva_clip.constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD from encoders_transformer import IDFormer, IDEncoder +from modules.errors import log + + +debug = log.trace if os.environ.get('SD_PULID_DEBUG', None) is not None else lambda *args, **kwargs: None class StableDiffusionXLPuLIDPipeline: @@ -33,14 +37,17 @@ class StableDiffusionXLPuLIDPipeline: self.sdp = sdp self.version = version self.folder = 'models--ToTheBeginning--PuLID' + debug(f'PulID init: device={self.device} dtype={self.dtype} dir={self.cache_dir} offload={self.offload} sdp={self.sdp} version={self.version}') - self.pipe.scheduler = DPMSolverMultistepScheduler.from_config(self.pipe.scheduler.config) + # self.pipe.scheduler = DPMSolverMultistepScheduler.from_config(self.pipe.scheduler.config) + self.hack_unet_attn_layers(self.pipe.unet) if self.version == 'v1.1': self.id_adapter = IDFormer().to(self.device, self.dtype) else: self.id_adapter = IDEncoder().to(self.device, self.dtype) + debug(f'PulID load: adapter={self.id_adapter.__class__.__name__}') self.providers = providers or ['CUDAExecutionProvider', 'CPUExecutionProvider'] - self.hack_unet_attn_layers(self.pipe.unet) + debug(f'PulID load: providers={self.providers}') # preprocessors # face align and parsing @@ -53,11 +60,13 @@ class StableDiffusionXLPuLIDPipeline: device=self.device, ) self.face_helper.face_parse = init_parsing_model(model_name='bisenet', device=self.device) + debug(f'PulID load: facehelper={self.face_helper.__class__.__name__}') # clip-vit backbone eva_precision = 'fp16' if self.dtype == torch.float16 or self.dtype == torch.bfloat16 else 'fp32' eva_model, _, _ = create_model_and_transforms('EVA02-CLIP-L-14-336', 'eva_clip', force_custom_clip=True, precision=eva_precision, device=self.device) self.clip_vision_model = eva_model.visual.to(dtype=self.dtype) + debug(f'PulID load: evaclip={self.clip_vision_model.__class__.__name__} precision={eva_precision}') eva_transform_mean = getattr(self.clip_vision_model, 'image_mean', OPENAI_DATASET_MEAN) eva_transform_std = getattr(self.clip_vision_model, 'image_std', OPENAI_DATASET_STD) if not isinstance(eva_transform_mean, (list, tuple)): @@ -75,9 +84,11 @@ class StableDiffusionXLPuLIDPipeline: root=os.path.join(self.cache_dir, self.folder), providers=self.providers, ) + debug(f'PulID load: faceanalysis={_loc}') self.app.prepare(ctx_id=0, det_size=(640, 640)) self.handler_ante = insightface.model_zoo.get_model(os.path.join(local_dir, 'glintr100.onnx')) self.handler_ante.prepare(ctx_id=0) + debug(f'PulID load: handler={self.handler_ante.__class__.__name__}') self.load_pretrain() @@ -150,6 +161,7 @@ class StableDiffusionXLPuLIDPipeline: ).to(unet.device, unet.dtype) else: id_adapter_attn_procs[name] = AttnProcessor() + debug(f'PulID attention: cls={IDAttnProcessor} std={AttnProcessor} len={len(id_adapter_attn_procs.keys())}') unet.set_attn_processor(id_adapter_attn_procs) self.id_adapter_attn_layers = nn.ModuleList(unet.attn_processors.values()) @@ -160,6 +172,7 @@ class StableDiffusionXLPuLIDPipeline: else: ckpt_path = hf_hub_download('guozinan/PuLID', 'pulid_v1.bin', local_dir=os.path.join(self.cache_dir, self.folder)) state_dict = torch.load(ckpt_path, map_location="cpu") + debug(f'PulID load: fn="{ckpt_path}"') state_dict_dict = {} for k, v in state_dict.items(): module = k.split('.')[0] @@ -255,6 +268,7 @@ class StableDiffusionXLPuLIDPipeline: self.clip_vision_model.to('cpu') # return id_embedding + debug(f'PulID embedding: cond={id_embedding.shape} uncond={uncond_id_embedding.shape}') return uncond_id_embedding, id_embedding def set_progress_bar_config(self, bar_format: str = None, ncols: int = 80, colour: str = None): @@ -264,9 +278,10 @@ class StableDiffusionXLPuLIDPipeline: pulid_sampling.trange = functools.partial(trange_orig, bar_format=bar_format, ncols=ncols, colour=colour) def sample(self, x, sigma, **extra_args): - x_ddim_space = x / (sigma[:, None, None, None] ** 2 + self.sigma_data**2) ** 0.5 t = self.timestep(sigma) + x_ddim_space = x / (sigma[:, None, None, None] ** 2 + self.sigma_data**2) ** 0.5 cfg_scale = extra_args['cfg_scale'] + debug(f'PulID sample start: step={self.step+1} x={x.shape} dtype={x.dtype} timestep={t.item()} sigma={sigma.shape} cfg={cfg_scale} args={extra_args.keys()}') eps_positive = self.pipe.unet(x_ddim_space, t, return_dict=False, **extra_args['positive'])[0] eps_negative = self.pipe.unet(x_ddim_space, t, return_dict=False, **extra_args['negative'])[0] noise_pred = eps_negative + cfg_scale * (eps_positive - eps_negative) @@ -274,6 +289,7 @@ class StableDiffusionXLPuLIDPipeline: if self.callback_on_step_end is not None: self.step += 1 self.callback_on_step_end(self.pipe, step=self.step, timestep=t, kwargs={ 'latents': latent }) + debug(f'PulID sample end: step={self.step} x={latent.shape} dtype={x.dtype} min={torch.amin(latent)} max={torch.amax(latent)}') return latent def init_latent(self, seed, size, image, mask_image, strength, width, height): # pylint: disable=unused-argument @@ -299,6 +315,7 @@ class StableDiffusionXLPuLIDPipeline: return_image_latents=False, ) latents = latents[0] + debug(f'PulID noise: op=inpaint latent={latents.shape} image={image} mask={mask_image} dtype={latents.dtype}') else: # img2img latents = self.pipe.prepare_latents(image, None, # timestep (not needed) @@ -309,10 +326,10 @@ class StableDiffusionXLPuLIDPipeline: None, # generator False, # add_noise ) - + debug(f'PulID noise: op=img2img latent={latents.shape} image={image} dtype={latents.dtype}') else: latents = torch.zeros_like(noise) - + debug(f'PulID noise: op=txt2img latent={latents.shape} dtype={latents.dtype}') return latents, noise def __call__( @@ -333,6 +350,7 @@ class StableDiffusionXLPuLIDPipeline: output_type: str='pil', callback_on_step_end=None, ): + debug(f'PulID call: width={width} height={height} cfg={guidance_scale} steps={num_inference_steps} seed={seed} strength={strength} id_scale={id_scale} output={output_type}') self.step = 0 # pylint: disable=attribute-defined-outside-init self.callback_on_step_end = callback_on_step_end # pylint: disable=attribute-defined-outside-init size = (1, height, width) @@ -341,11 +359,12 @@ class StableDiffusionXLPuLIDPipeline: if image is not None and strength > 0: _, num_inference_steps = self.pipe.get_timesteps(num_inference_steps, strength, self.device, None) # denoising_start disabled sigmas = sigmas[-(num_inference_steps + 1):].to(self.device) # shorten sigmas in i2i - + debug(f'PulID sigmas: sigmas={sigmas.shape} dtype={sigmas.dtype}') # latents latent, noise = self.init_latent(seed, size, image, mask_image, strength, width, height) noisy_latent = latent + noise * sigmas[0].to(noise) + debug(f'PulID noisy: latent={noisy_latent.shape} dtype={noisy_latent.dtype}') ( prompt_embeds, @@ -390,14 +409,17 @@ class StableDiffusionXLPuLIDPipeline: latents = self.sampler(self.sample, noisy_latent, sigmas, extra_args=sampler_kwargs, disable=False, mask_args=mask_args) # process output + latents = latents.to(dtype=self.pipe.vae.dtype, device=self.device) + debug(f'PulID output: latent={latents.shape} dtype={latents.dtype}') if output_type == 'latent': images = self.pipe.image_processor.postprocess(latents, output_type='latent') elif output_type == 'np': images = self.pipe.image_processor.postprocess(latents, output_type='np') else: - latents = latents.to(dtype=self.pipe.vae.dtype, device=self.device) / self.pipe.vae.config.scaling_factor + latents = latents / self.pipe.vae.config.scaling_factor images = self.pipe.vae.decode(latents).sample images = self.pipe.image_processor.postprocess(images, output_type='pil') + debug(f'PulID output: type={type(images)} images={images.shape if hasattr(images, "shape") else images}') return StableDiffusionXLPipelineOutput(images) From 70eec4349ecc9dc38c60a22d55dd2a0952406414 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 11 Nov 2024 10:46:33 -0500 Subject: [PATCH 35/40] fix pulid vae Signed-off-by: Vladimir Mandic --- modules/processing_vae.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 5e6fa68f4..0473beeb1 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -147,6 +147,7 @@ def taesd_vae_encode(image): def vae_decode(latents, model, output_type='np', full_quality=True, width=None, height=None): t0 = time.time() + model = model or shared.sd_model if latents is None or not torch.is_tensor(latents): # already decoded return latents prev_job = shared.state.job @@ -169,15 +170,8 @@ def vae_decode(latents, model, output_type='np', full_quality=True, width=None, if latents.shape[-1] <= 4: # not a latent, likely an image decoded = latents.float().cpu().numpy() - elif full_quality and hasattr(shared.sd_model, "vae"): - parent = shared.sd_model if hasattr(shared.sd_model, 'vae') else None - if hasattr(shared.sd_model, 'vae'): - parent = shared.sd_model - elif hasattr(shared.sd_model, 'pipe') and hasattr(shared.sd_model.pipe, 'vae'): - parent = shared.sd_model.pipe - else: - parent = None - decoded = full_vae_decode(latents=latents, model=parent) + elif full_quality and hasattr(model, "vae"): + decoded = full_vae_decode(latents=latents, model=model) else: decoded = taesd_vae_decode(latents=latents) From e2c7c8cf2e2c4996b04685c03f036b4dc7a963b3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 11 Nov 2024 15:23:11 -0500 Subject: [PATCH 36/40] refactor processing class Signed-off-by: Vladimir Mandic --- .../Lora/extra_networks_lora.py | 1 - modules/api/models.py | 8 - modules/img2img.py | 19 +- modules/processing_class.py | 449 +++++++++--------- modules/shared.py | 6 +- modules/txt2img.py | 1 - modules/ui_img2img.py | 2 + modules/unipc/sampler.py | 2 +- 8 files changed, 243 insertions(+), 245 deletions(-) diff --git a/extensions-builtin/Lora/extra_networks_lora.py b/extensions-builtin/Lora/extra_networks_lora.py index 9172d7336..69b234df7 100644 --- a/extensions-builtin/Lora/extra_networks_lora.py +++ b/extensions-builtin/Lora/extra_networks_lora.py @@ -145,7 +145,6 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): if self.active and networks.debug: shared.log.debug(f"Network end: type=LoRA load={networks.timer['load']:.2f} apply={networks.timer['apply']:.2f} restore={networks.timer['restore']:.2f}") if self.errors: - p.comment("Networks with errors: " + ", ".join(f"{k} ({v})" for k, v in self.errors.items())) for k, v in self.errors.items(): shared.log.error(f'LoRA: name="{k}" errors={v}') self.errors.clear() diff --git a/modules/api/models.py b/modules/api/models.py index 740f3c555..f5b89c2a9 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -11,14 +11,6 @@ API_NOT_ALLOWED = [ "sd_model", "outpath_samples", "outpath_grids", - "sampler_index", - "extra_generation_params", - "overlay_images", - "do_not_reload_embeddings", - "seed_enable_extras", - "prompt_for_display", - "sampler_noise_scheduler_override", - "ddim_discretize" ] class ModelDef(BaseModel): diff --git a/modules/img2img.py b/modules/img2img.py index f3bec5b02..8274386cc 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -137,6 +137,7 @@ def img2img(id_task: str, state: str, mode: int, inpaint_full_res, inpaint_full_res_padding, inpainting_mask_invert, img2img_batch_files, img2img_batch_input_dir, img2img_batch_output_dir, img2img_batch_inpaint_mask_dir, hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio, + enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, hr_refiner_start, refiner_prompt, refiner_negative, override_settings_texts, *args): # pylint: disable=unused-argument @@ -214,7 +215,6 @@ def img2img(id_task: str, state: str, mode: int, subseed_strength=subseed_strength, seed_resize_from_h=seed_resize_from_h, seed_resize_from_w=seed_resize_from_w, - seed_enable_extras=True, sampler_name = processing.get_sampler_name(sampler_index, img=True), batch_size=batch_size, n_iter=n_iter, @@ -247,6 +247,23 @@ def img2img(id_task: str, state: str, mode: int, inpainting_mask_invert=inpainting_mask_invert, hdr_mode=hdr_mode, hdr_brightness=hdr_brightness, hdr_color=hdr_color, hdr_sharpen=hdr_sharpen, hdr_clamp=hdr_clamp, hdr_boundary=hdr_boundary, hdr_threshold=hdr_threshold, hdr_maximize=hdr_maximize, hdr_max_center=hdr_max_center, hdr_max_boundry=hdr_max_boundry, hdr_color_picker=hdr_color_picker, hdr_tint_ratio=hdr_tint_ratio, + # refiner + enable_hr=enable_hr, + hr_denoising_strength=hr_denoising_strength, + hr_scale=hr_scale, + hr_resize_mode=hr_resize_mode, + hr_resize_context=hr_resize_context, + hr_upscaler=hr_upscaler, + hr_force=hr_force, + hr_second_pass_steps=hr_second_pass_steps, + hr_resize_x=hr_resize_x, + hr_resize_y=hr_resize_y, + hr_sampler_name = processing.get_sampler_name(hr_sampler_index), + refiner_steps=refiner_steps, + hr_refiner_start=hr_refiner_start, + refiner_prompt=refiner_prompt, + refiner_negative=refiner_negative, + # override override_settings=override_settings, ) p.scripts = modules.scripts.scripts_img2img diff --git a/modules/processing_class.py b/modules/processing_class.py index 2f11db375..215507b4a 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -17,53 +17,44 @@ debug = shared.log.trace if os.environ.get('SD_PROCESS_DEBUG', None) is not None @dataclass(repr=False) class StableDiffusionProcessing: - """ - The first set of paramaters: sd_models -> do_not_reload_embeddings represent the minimum required to create a StableDiffusionProcessing - """ def __init__(self, - sd_model=None, - outpath_samples=None, - outpath_grids=None, + sd_model=None, # pylint: disable=unused-argument # local instance of sd_model + # base params prompt: str = "", - styles: List[str] = None, + negative_prompt: str = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, - seed_enable_extras: bool = True, - sampler_name: str = None, - hr_sampler_name: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, - cfg_scale: float = 7.0, - image_cfg_scale: float = None, clip_skip: int = 1, width: int = 512, height: int = 512, - full_quality: bool = True, - detailer: bool = False, - restore_faces: bool = False, - tiling: bool = False, - hidiffusion: bool = False, - do_not_save_samples: bool = False, - do_not_save_grid: bool = False, - extra_generation_params: Dict[Any, Any] = None, - overlay_images: Any = None, - negative_prompt: str = None, + # samplers + sampler_index: int = None, # pylint: disable=unused-argument # used only to set sampler_name + sampler_name: str = None, + hr_sampler_name: str = None, eta: float = None, - do_not_reload_embeddings: bool = False, - denoising_strength: float = 0, + # guidance + cfg_scale: float = 7.0, + cfg_end: float = 1, diffusers_guidance_rescale: float = 0.7, pag_scale: float = 0.0, pag_adaptive: float = 0.5, - cfg_end: float = 1, - resize_mode: int = 0, - resize_name: str = 'None', - resize_context: str = 'None', - scale_by: float = 0, - selected_scale_tab: int = 0, + # styles + styles: List[str] = None, + # vae + tiling: bool = False, + full_quality: bool = True, + # other + hidiffusion: bool = False, + do_not_reload_embeddings: bool = False, + detailer: bool = False, + restore_faces: bool = False, + # hdr corrections hdr_mode: int = 0, hdr_brightness: float = 0, hdr_color: float = 0, @@ -76,92 +67,209 @@ class StableDiffusionProcessing: hdr_max_boundry: float = 1.0, hdr_color_picker: str = None, hdr_tint_ratio: float = 0, - override_settings: Dict[str, Any] = None, + # img2img + init_images: list = None, + init_latent: Any = None, + resize_mode: int = 0, + resize_name: str = 'None', + resize_context: str = 'None', + denoising_strength: float = 0.3, + image_cfg_scale: float = None, + initial_noise_multiplier: float = None, # pylint: disable=unused-argument # a1111 compatibility + scale_by: float = 1, + selected_scale_tab: int = 0, # pylint: disable=unused-argument # a1111 compatibility + # inpaint + mask: Any = None, + image_mask: Any = None, + latent_mask: Any = None, + mask_for_overlay: Any = None, + mask_blur: int = 4, + paste_to: Any = None, + inpainting_fill: int = 0, + inpaint_full_res: bool = False, + inpaint_full_res_padding: int = 0, + inpainting_mask_invert: int = 0, + overlay_images: Any = None, + # refiner + enable_hr: bool = False, + firstphase_width: int = 0, + firstphase_height: int = 0, + hr_scale: float = 2.0, + hr_force: bool = False, + hr_resize_mode: int = 0, + hr_resize_context: str = 'None', + hr_upscaler: str = None, + hr_second_pass_steps: int = 0, + hr_resize_x: int = 0, + hr_resize_y: int = 0, + hr_denoising_strength: float = 0.50, + refiner_steps: int = 5, + refiner_start: float = 0, + refiner_prompt: str = '', + refiner_negative: str = '', + hr_refiner_start: float = 0, + # save options + outpath_samples=None, + outpath_grids=None, + do_not_save_samples: bool = False, + do_not_save_grid: bool = False, + # scripts + script_args: list = [], + # overrides + override_settings: Dict[str, Any] = {}, override_settings_restore_afterwards: bool = True, - sampler_index: int = None, - script_args: list = None - ): # pylint: disable=unused-argument - + # metadata + extra_generation_params: Dict[Any, Any] = {}, + ): + self.task_args = {} + # state items self.state: str = '' + self.ops = [] self.skip = [] - self.outpath_samples: str = outpath_samples - self.outpath_grids: str = outpath_grids - self.prompt: str = prompt - self.prompt_for_display: str = None - self.negative_prompt: str = (negative_prompt or "") - self.styles: list = styles or [] - self.seed: int = seed - self.subseed: int = subseed - self.subseed_strength: float = subseed_strength - self.seed_resize_from_h: int = seed_resize_from_h - self.seed_resize_from_w: int = seed_resize_from_w - self.sampler_name: str = sampler_name - self.hr_sampler_name: str = hr_sampler_name if hr_sampler_name != 'Same as primary' else sampler_name - self.batch_size: int = batch_size - self.n_iter: int = n_iter - self.steps: int = steps - self.hr_second_pass_steps = 0 - self.cfg_scale: float = cfg_scale - self.scale_by: float = scale_by + self.color_corrections = [] + self.is_control = False + self.is_hr_pass = False + self.is_refiner_pass = False + self.is_api = False + self.scheduled_prompt = False + self.prompt_embeds = [] + self.positive_pooleds = [] + self.negative_embeds = [] + self.negative_pooleds = [] + self.disable_extra_networks = False + self.iteration = 0 + # initializers + self.prompt = prompt + self.seed = seed + self.subseed = subseed + self.subseed_strength = subseed_strength + self.seed_resize_from_h = seed_resize_from_h + self.seed_resize_from_w = seed_resize_from_w + self.batch_size = batch_size + self.n_iter = n_iter + self.steps = steps + self.clip_skip = clip_skip + self.width = width + self.height = height + self.negative_prompt = negative_prompt + self.styles = styles + self.tiling = tiling + self.full_quality = full_quality + self.hidiffusion = hidiffusion + self.do_not_reload_embeddings = do_not_reload_embeddings + self.detailer = detailer + self.restore_faces = restore_faces + self.hdr_mode = hdr_mode + self.hdr_brightness = hdr_brightness + self.hdr_color = hdr_color + self.hdr_sharpen = hdr_sharpen + self.hdr_clamp = hdr_clamp + self.hdr_boundary = hdr_boundary + self.hdr_threshold = hdr_threshold + self.hdr_maximize = hdr_maximize + self.hdr_max_center = hdr_max_center + self.hdr_max_boundry = hdr_max_boundry + self.hdr_color_picker = hdr_color_picker + self.hdr_tint_ratio = hdr_tint_ratio + self.init_images = init_images + self.resize_mode = resize_mode + self.resize_name = resize_name + self.resize_context = resize_context + self.denoising_strength = denoising_strength self.image_cfg_scale = image_cfg_scale + self.scale_by = scale_by + self.mask = mask + self.image_mask = mask + self.latent_mask = latent_mask + self.mask_blur = mask_blur + self.inpainting_fill = inpainting_fill + self.inpaint_full_res_padding = inpaint_full_res_padding + self.inpainting_mask_invert = inpainting_mask_invert + self.overlay_images = overlay_images + self.enable_hr = enable_hr + self.firstphase_width = firstphase_width + self.firstphase_height = firstphase_height + self.hr_scale = hr_scale + self.hr_force = hr_force + self.hr_resize_mode = hr_resize_mode + self.hr_resize_context = hr_resize_context + self.hr_upscaler = hr_upscaler + self.hr_second_pass_steps = hr_second_pass_steps + self.hr_resize_x = hr_resize_x + self.hr_resize_y = hr_resize_y + self.hr_upscale_to_x = hr_resize_x + self.hr_upscale_to_y = hr_resize_y + self.hr_denoising_strength = hr_denoising_strength + self.refiner_steps = refiner_steps + self.refiner_start = refiner_start + self.refiner_prompt = refiner_prompt + self.refiner_negative = refiner_negative + self.hr_refiner_start = hr_refiner_start + self.outpath_samples = outpath_samples + self.outpath_grids = outpath_grids + self.do_not_save_samples = do_not_save_samples + self.do_not_save_grid = do_not_save_grid + self.override_settings_restore_afterwards = override_settings_restore_afterwards + self.extra_generation_params = extra_generation_params + self.eta = eta + self.cfg_scale = cfg_scale + self.cfg_end = cfg_end self.diffusers_guidance_rescale = diffusers_guidance_rescale self.pag_scale = pag_scale self.pag_adaptive = pag_adaptive - self.cfg_end = cfg_end - self.width: int = width - self.height: int = height - self.full_quality: bool = full_quality - self.detailer: bool = detailer - self.restore_faces: bool = restore_faces - self.tiling: bool = tiling - self.hidiffusion: bool = hidiffusion - self.do_not_save_samples: bool = do_not_save_samples - self.do_not_save_grid: bool = do_not_save_grid - self.extra_generation_params: dict = extra_generation_params or {} - self.overlay_images = overlay_images - self.eta = eta - self.do_not_reload_embeddings = do_not_reload_embeddings - self.paste_to = None - self.color_corrections = None - self.denoising_strength: float = denoising_strength + self.selected_scale_tab = selected_scale_tab + self.mask_for_overlay = mask_for_overlay + self.paste_to = paste_to + self.init_latent = None + # special handled items + if firstphase_width != 0 or firstphase_height != 0: + self.hr_upscale_to_x = self.width + self.hr_upscale_to_y = self.height + self.width = firstphase_width + self.height = firstphase_height + self.sampler_name = sampler_name or processing_helpers.get_sampler_name(sampler_index, img=True) + self.hr_sampler_name: str = hr_sampler_name if hr_sampler_name != 'Same as primary' else self.sampler_name self.override_settings = {k: v for k, v in (override_settings or {}).items() if k not in shared.restricted_opts} - self.override_settings_restore_afterwards = override_settings_restore_afterwards - self.is_using_inpainting_conditioning = False # a111 compatibility - self.disable_extra_networks = False - # self.scripts = scripts.ScriptRunner() # set via property - # self.script_args = script_args or [] # set via property - self.per_script_args = {} + self.inpaint_full_res = inpaint_full_res if isinstance(inpaint_full_res, bool) else self.inpaint_full_res + self.inpaint_full_res = inpaint_full_res != 0 if isinstance(inpaint_full_res, int) else self.inpaint_full_res + + # null items initialized later self.all_prompts = None self.all_negative_prompts = None self.all_seeds = None self.all_subseeds = None - self.clip_skip = clip_skip + # ip adapter + self.ip_adapter_names = [] + self.ip_adapter_scales = [0.0] + self.ip_adapter_images = [] + self.ip_adapter_starts = [0.0] + self.ip_adapter_ends = [1.0] + self.ip_adapter_crops = [] + # a1111 compatibility items shared.opts.data['clip_skip'] = int(self.clip_skip) # for compatibility with a1111 sd_hijack_clip - self.iteration = 0 - self.is_control = False - self.is_hr_pass = False - self.is_refiner_pass = False - self.hr_force = False - self.enable_hr = None - self.hr_scale = None - self.hr_upscaler = None - self.hr_resize_mode = 0 - self.hr_resize_context = 'None' - self.hr_resize_x = 0 - self.hr_resize_y = 0 - self.hr_upscale_to_x = 0 - self.hr_upscale_to_y = 0 + self.seed_enable_extras: bool = True, + self.is_using_inpainting_conditioning = False # a111 compatibility + self.batch_index = 0 + self.refiner_switch_at = 0 + self.hr_prompt = '' + self.all_hr_prompts = [] + self.hr_negative_prompt = '' + self.all_hr_negative_prompts = [] self.truncate_x = 0 self.truncate_y = 0 - self.applied_old_hires_behavior_to = None - self.refiner_steps = 5 - self.refiner_start = 0 - self.refiner_prompt = '' - self.refiner_negative = '' - self.ops = [] - self.resize_mode: int = resize_mode - self.resize_name: str = resize_name - self.resize_context: str = resize_context + self.comments = {} + self.sampler = None + self.nmask = None + self.initial_noise_multiplier = initial_noise_multiplier or shared.opts.initial_noise_multiplier + self.image_conditioning = None + self.prompt_for_display: str = None + # scripts + self.scripts_value: scripts.ScriptRunner = field(default=None, init=False) + self.script_args_value: list = field(default=None, init=False) + self.scripts_setup_complete: bool = field(default=False, init=False) + self.script_args = script_args + self.per_script_args = {} + # settings to processing self.ddim_discretize = shared.opts.ddim_discretize self.s_min_uncond = shared.opts.s_min_uncond self.s_churn = shared.opts.s_churn @@ -205,11 +313,11 @@ class StableDiffusionProcessing: self.hdr_tint_ratio=hdr_tint_ratio # globals self.embedder = None - # self.scheduled_prompt: bool = False - # self.prompt_embeds = [] - # self.positive_pooleds = [] - # self.negative_embeds = [] - # self.negative_pooleds = [] + self.scheduled_prompt: bool = False + self.prompt_embeds = [] + self.positive_pooleds = [] + self.negative_embeds = [] + self.negative_pooleds = [] @property def sd_model(self): @@ -253,57 +361,9 @@ class StableDiffusionProcessing: class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): - - def __init__(self, - enable_hr: bool = False, - denoising_strength: float = 0.75, - firstphase_width: int = 0, - firstphase_height: int = 0, - hr_scale: float = 2.0, - hr_force: bool = False, - hr_resize_mode: int = 0, - hr_resize_context: str = 'None', - hr_upscaler: str = None, - hr_second_pass_steps: int = 0, - hr_resize_x: int = 0, - hr_resize_y: int = 0, - refiner_steps: int = 5, - refiner_start: float = 0, - refiner_prompt: str = '', - refiner_negative: str = '', - **kwargs - ): - + def __init__(self, **kwargs): + debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access super().__init__(**kwargs) - self.reprocess = {} - self.enable_hr = enable_hr - self.denoising_strength = denoising_strength - self.hr_scale = hr_scale - self.hr_upscaler = hr_upscaler - self.hr_resize_mode = hr_resize_mode - self.hr_resize_context = hr_resize_context - self.hr_force = hr_force - self.hr_second_pass_steps = hr_second_pass_steps - self.hr_resize_x = hr_resize_x - self.hr_resize_y = hr_resize_y - self.hr_upscale_to_x = hr_resize_x - self.hr_upscale_to_y = hr_resize_y - if firstphase_width != 0 or firstphase_height != 0: - self.hr_upscale_to_x = self.width - self.hr_upscale_to_y = self.height - self.width = firstphase_width - self.height = firstphase_height - self.truncate_x = 0 - self.truncate_y = 0 - self.applied_old_hires_behavior_to = None - self.refiner_steps = refiner_steps - self.refiner_start = refiner_start - self.refiner_prompt = refiner_prompt - self.refiner_negative = refiner_negative - self.sampler = None - self.scripts = None - self.script_args = [] - def init(self, all_prompts=None, all_seeds=None, all_subseeds=None): if shared.native: @@ -361,41 +421,9 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): - - def __init__(self, init_images: list = None, resize_mode: int = 0, resize_name: str = 'None', resize_context: str = 'None', denoising_strength: float = 0.3, image_cfg_scale: float = None, mask: Any = None, mask_blur: int = 4, inpainting_fill: int = 0, inpaint_full_res: bool = False, inpaint_full_res_padding: int = 0, inpainting_mask_invert: int = 0, initial_noise_multiplier: float = None, scale_by: float = 1, refiner_steps: int = 5, refiner_start: float = 0, refiner_prompt: str = '', refiner_negative: str = '', **kwargs): + def __init__(self, **kwargs): + debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access super().__init__(**kwargs) - self.init_images = init_images - self.resize_mode: int = resize_mode - self.resize_name: str = resize_name - self.resize_context: str = resize_context - self.denoising_strength: float = denoising_strength - self.hr_denoising_strength: float = denoising_strength - self.image_cfg_scale: float = image_cfg_scale - self.init_latent = None - self.image_mask = mask - self.latent_mask = None - self.mask_for_overlay = None - self.mask_blur_x = mask_blur # a1111 compatibility item - self.mask_blur_y = mask_blur # a1111 compatibility item - self.mask_blur = mask_blur - self.inpainting_fill = inpainting_fill - self.inpaint_full_res = inpaint_full_res - self.inpaint_full_res_padding = inpaint_full_res_padding - self.inpainting_mask_invert = inpainting_mask_invert - self.initial_noise_multiplier = shared.opts.initial_noise_multiplier if initial_noise_multiplier is None else initial_noise_multiplier - self.mask = None - self.nmask = None - self.image_conditioning = None - self.refiner_steps = refiner_steps - self.refiner_start = refiner_start - self.refiner_prompt = refiner_prompt - self.refiner_negative = refiner_negative - self.enable_hr = None - self.is_batch = False - self.scale_by = scale_by - self.sampler = None - self.scripts = None - self.script_args = [] def init(self, all_prompts=None, all_seeds=None, all_subseeds=None): if hasattr(self, 'init_images') and self.init_images is not None and len(self.init_images) > 0: @@ -545,47 +573,8 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): class StableDiffusionProcessingControl(StableDiffusionProcessingImg2Img): def __init__(self, **kwargs): + debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access super().__init__(**kwargs) - self.strength = None - self.adapter_conditioning_scale = None - self.adapter_conditioning_factor = None - self.guess_mode = None - self.controlnet_conditioning_scale = None - self.control_guidance_start = None - self.control_guidance_end = None - self.control_mode = None - self.reference_attn = None - self.reference_adain = None - self.attention_auto_machine_weight = None - self.gn_auto_machine_weight = None - self.style_fidelity = None - self.ref_image = None - self.image = None - self.query_weight = None - self.adain_weight = None - self.adapter_conditioning_factor = 1.0 - self.attention = 'Attention' - self.fidelity = 0.5 - self.mask_image = None - self.override = None - self.resize_mode_before = None - self.resize_name_before = None - self.width_before = None - self.height_before = None - self.scale_by_before = None - self.selected_scale_tab_before = None - self.resize_mode_after = None - self.resize_name_after = None - self.width_after = None - self.height_after = None - self.scale_by_after = None - self.selected_scale_tab_after = None - self.resize_mode_mask = None - self.resize_name_mask = None - self.width_mask = None - self.height_mask = None - self.scale_by_mask = None - self.selected_scale_tab_mask = None def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): # abstract pass diff --git a/modules/shared.py b/modules/shared.py index 48984c8be..7d19c721e 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -822,8 +822,8 @@ options_templates.update(options_section(('postprocessing', "Postprocessing"), { "mask_apply_overlay": OptionInfo(True, "Apply mask as overlay"), "img2img_background_color": OptionInfo("#ffffff", "Image transparent color fill", gr.ColorPicker, {}), "inpainting_mask_weight": OptionInfo(1.0, "Inpainting conditioning mask strength", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - "initial_noise_multiplier": OptionInfo(1.0, "Noise multiplier for image processing", gr.Slider, {"minimum": 0.1, "maximum": 1.5, "step": 0.01}), - "img2img_extra_noise": OptionInfo(0.0, "Extra noise multiplier for img2img", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + "initial_noise_multiplier": OptionInfo(1.0, "Noise multiplier for image processing", gr.Slider, {"minimum": 0.1, "maximum": 1.5, "step": 0.01, "visible": not native}), + "img2img_extra_noise": OptionInfo(0.0, "Extra noise multiplier for img2img", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01, "visible": not native}), # "postprocessing_sep_detailer": OptionInfo("

Detailer

", "", gr.HTML), "detailer_model": OptionInfo("Detailer", "Detailer model", gr.Radio, lambda: {"choices": [x.name() for x in detailers], "visible": False}), @@ -1148,7 +1148,7 @@ cmd_opts = cmd_args.settings_args(opts, cmd_opts) if cmd_opts.use_xformers: opts.data['cross_attention_optimization'] = 'xFormers' opts.data['uni_pc_lower_order_final'] = opts.schedulers_use_loworder # compatibility -opts.data['uni_pc_order'] = opts.schedulers_solver_order # compatibility +opts.data['uni_pc_order'] = max(2, opts.schedulers_solver_order) # compatibility log.info(f'Engine: backend={backend} compute={devices.backend} device={devices.get_optimal_device_name()} attention="{opts.cross_attention_optimization}" mode={devices.inference_context.__name__}') if not native: log.warning('Backend=original is in maintainance-only mode') diff --git a/modules/txt2img.py b/modules/txt2img.py index 38cde0aca..2f0e2f4b3 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -49,7 +49,6 @@ def txt2img(id_task, state, subseed_strength=subseed_strength, seed_resize_from_h=seed_resize_from_h, seed_resize_from_w=seed_resize_from_w, - seed_enable_extras=True, sampler_name = processing.get_sampler_name(sampler_index), hr_sampler_name = processing.get_sampler_name(hr_sampler_index), batch_size=batch_size, diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index 4cb8e4c18..22c89dac8 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -131,6 +131,7 @@ def create_ui(): full_quality, tiling, hidiffusion, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_sections.create_advanced_inputs('img2img') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('img2img') + enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, hr_refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') detailer = shared.yolo.ui('img2img') # with gr.Group(elem_id="inpaint_controls", visible=False) as inpaint_controls: @@ -192,6 +193,7 @@ def create_ui(): inpaint_full_res, inpaint_full_res_padding, inpainting_mask_invert, img2img_batch_files, img2img_batch_input_dir, img2img_batch_output_dir, img2img_batch_inpaint_mask_dir, hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio, + enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, hr_refiner_start, refiner_prompt, refiner_negative, override_settings, ] img2img_dict = dict( diff --git a/modules/unipc/sampler.py b/modules/unipc/sampler.py index bcc4eed76..b5e116d61 100644 --- a/modules/unipc/sampler.py +++ b/modules/unipc/sampler.py @@ -186,6 +186,6 @@ class UniPCSampler(object): ) uni_pc = UniPC(model_fn, self.noise_schedule, predict_x0=True, thresholding=False, variant=shared.opts.uni_pc_variant, condition=conditioning, unconditional_condition=unconditional_conditioning, before_sample=self.before_sample, after_sample=self.after_sample, after_update=self.after_update) - x = uni_pc.sample(img, steps=S, skip_type=shared.opts.uni_pc_skip_type, method="multistep", order=shared.opts.schedulers_solver_order, lower_order_final=shared.opts.schedulers_use_loworder) + x = uni_pc.sample(img, steps=S, skip_type=shared.opts.uni_pc_skip_type, method="multistep", order=shared.opts.uni_pc_order, lower_order_final=shared.opts.uni_pc_lower_order_final) return x.to(device), None From 910b88e632f00e9ffcfc3f37858ffb4ec5dca86e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 11 Nov 2024 15:51:30 -0500 Subject: [PATCH 37/40] add refine/hires to img2img Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 ++ modules/control/run.py | 2 +- modules/processing_class.py | 9 ++++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5db77983c..f4178db22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,8 @@ This release can be considered an LTS release before we kick off the next round - add show networks on startup setting - better mapping of networks previews - optimize networks display load + - Image2image: + - integrated refine/upscale/hires workflow - Other: - Installer: - Log `venv` and package search paths diff --git a/modules/control/run.py b/modules/control/run.py index 74bab35c4..d2e1b36ef 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -70,7 +70,7 @@ def control_run(state: str = '', enable_hr: bool = False, hr_sampler_index: int = None, hr_denoising_strength: float = 0.3, hr_resize_mode: int = 0, hr_resize_context: str = 'None', hr_upscaler: str = None, hr_force: bool = False, hr_second_pass_steps: int = 20, hr_scale: float = 1.0, hr_resize_x: int = 0, hr_resize_y: int = 0, refiner_steps: int = 5, refiner_start: float = 0.0, refiner_prompt: str = '', refiner_negative: str = '', video_skip_frames: int = 0, video_type: str = 'None', video_duration: float = 2.0, video_loop: bool = False, video_pad: int = 0, video_interpolate: int = 0, - *input_script_args + *input_script_args, ): # handle optional initialization via ui for u in units: diff --git a/modules/processing_class.py b/modules/processing_class.py index 215507b4a..333e11925 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -69,7 +69,6 @@ class StableDiffusionProcessing: hdr_tint_ratio: float = 0, # img2img init_images: list = None, - init_latent: Any = None, resize_mode: int = 0, resize_name: str = 'None', resize_context: str = 'None', @@ -80,7 +79,6 @@ class StableDiffusionProcessing: selected_scale_tab: int = 0, # pylint: disable=unused-argument # a1111 compatibility # inpaint mask: Any = None, - image_mask: Any = None, latent_mask: Any = None, mask_for_overlay: Any = None, mask_blur: int = 4, @@ -179,7 +177,7 @@ class StableDiffusionProcessing: self.image_cfg_scale = image_cfg_scale self.scale_by = scale_by self.mask = mask - self.image_mask = mask + self.image_mask = mask # TODO duplciate mask params self.latent_mask = latent_mask self.mask_blur = mask_blur self.inpainting_fill = inpainting_fill @@ -221,6 +219,7 @@ class StableDiffusionProcessing: self.mask_for_overlay = mask_for_overlay self.paste_to = paste_to self.init_latent = None + # special handled items if firstphase_width != 0 or firstphase_height != 0: self.hr_upscale_to_x = self.width @@ -238,6 +237,7 @@ class StableDiffusionProcessing: self.all_negative_prompts = None self.all_seeds = None self.all_subseeds = None + # ip adapter self.ip_adapter_names = [] self.ip_adapter_scales = [0.0] @@ -245,6 +245,7 @@ class StableDiffusionProcessing: self.ip_adapter_starts = [0.0] self.ip_adapter_ends = [1.0] self.ip_adapter_crops = [] + # a1111 compatibility items shared.opts.data['clip_skip'] = int(self.clip_skip) # for compatibility with a1111 sd_hijack_clip self.seed_enable_extras: bool = True, @@ -263,12 +264,14 @@ class StableDiffusionProcessing: self.initial_noise_multiplier = initial_noise_multiplier or shared.opts.initial_noise_multiplier self.image_conditioning = None self.prompt_for_display: str = None + # scripts self.scripts_value: scripts.ScriptRunner = field(default=None, init=False) self.script_args_value: list = field(default=None, init=False) self.scripts_setup_complete: bool = field(default=False, init=False) self.script_args = script_args self.per_script_args = {} + # settings to processing self.ddim_discretize = shared.opts.ddim_discretize self.s_min_uncond = shared.opts.s_min_uncond From b8cbe10c836995719582827acc0b6dde8728ce09 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 11 Nov 2024 16:16:22 -0500 Subject: [PATCH 38/40] add bnb and quanto version info Signed-off-by: Vladimir Mandic --- modules/model_quant.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/model_quant.py b/modules/model_quant.py index 547a3d7ae..68bdfa7b2 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -32,14 +32,14 @@ def load_bnb(msg='', silent=False): global bnb # pylint: disable=global-statement if bnb is not None: return bnb - fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access - log.debug(f'Quantization: type=bitsandbytes fn={fn}') # pylint: disable=protected-access install('bitsandbytes', quiet=True) try: import bitsandbytes bnb = bitsandbytes diffusers.utils.import_utils._bitsandbytes_available = True # pylint: disable=protected-access diffusers.utils.import_utils._bitsandbytes_version = '0.43.3' # pylint: disable=protected-access + fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access + log.debug(f'Quantization: type=bitsandbytes version={bnb.__version__} fn={fn}') # pylint: disable=protected-access return bnb except Exception as e: if len(msg) > 0: @@ -54,12 +54,12 @@ def load_quanto(msg='', silent=False): global quanto # pylint: disable=global-statement if quanto is not None: return quanto - fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access - log.debug(f'Quantization: type=quanto fn={fn}') # pylint: disable=protected-access install('optimum-quanto', quiet=True) try: from optimum import quanto as optimum_quanto # pylint: disable=no-name-in-module quanto = optimum_quanto + fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access + log.debug(f'Quantization: type=quanto version={quanto.__version__} fn={fn}') # pylint: disable=protected-access return quanto except Exception as e: if len(msg) > 0: From f590445cd665ca62772148515e0cafb661ea0e8b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 11 Nov 2024 16:38:37 -0500 Subject: [PATCH 39/40] img2img refine upscale Signed-off-by: Vladimir Mandic --- modules/processing_diffusers.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 7537d0209..76009d2e1 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -152,13 +152,14 @@ def process_hires(p: processing.StableDiffusionProcessing, output): p.is_hr_pass = True if hasattr(p, 'init_hr'): p.init_hr(p.hr_scale, p.hr_upscaler, force=p.hr_force) - else: # fake hires for img2img - p.hr_scale = p.scale_by - p.hr_upscaler = p.resize_name - p.hr_resize_mode = p.resize_mode - p.hr_resize_context = p.resize_context - p.hr_upscale_to_x = p.width - p.hr_upscale_to_y = p.height + else: + if not p.is_hr_pass: # fake hires for img2img if not actual hr pass + p.hr_scale = p.scale_by + p.hr_upscaler = p.resize_name + p.hr_resize_mode = p.resize_mode + p.hr_resize_context = p.resize_context + p.hr_upscale_to_x = p.width * p.hr_scale if p.hr_resize_x == 0 else p.hr_resize_x + p.hr_upscale_to_y = p.height * p.hr_scale if p.hr_resize_y == 0 else p.hr_resize_y prev_job = shared.state.job # hires runs on original pipeline From 8b3d99535db450fbaea119d64a9273c9d87687d4 Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Mon, 11 Nov 2024 19:18:38 -0600 Subject: [PATCH 40/40] lora loading for wrapped models (pulid) --- extensions-builtin/Lora/networks.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index 160487e88..b227f82f2 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -50,44 +50,45 @@ convert_diffusers_name_to_compvis = lora_convert.convert_diffusers_name_to_compv def assign_network_names_to_compvis_modules(sd_model): if sd_model is None: return + sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility network_layer_mapping = {} if shared.native: - if hasattr(shared.sd_model, 'text_encoder') and shared.sd_model.text_encoder is not None: - for name, module in shared.sd_model.text_encoder.named_modules(): - prefix = "lora_te1_" if hasattr(shared.sd_model, 'text_encoder_2') else "lora_te_" + if hasattr(sd_model, 'text_encoder') and sd_model.text_encoder is not None: + for name, module in sd_model.text_encoder.named_modules(): + prefix = "lora_te1_" if hasattr(sd_model, 'text_encoder_2') else "lora_te_" network_name = prefix + name.replace(".", "_") network_layer_mapping[network_name] = module module.network_layer_name = network_name - if hasattr(shared.sd_model, 'text_encoder_2'): - for name, module in shared.sd_model.text_encoder_2.named_modules(): + if hasattr(sd_model, 'text_encoder_2'): + for name, module in sd_model.text_encoder_2.named_modules(): network_name = "lora_te2_" + name.replace(".", "_") network_layer_mapping[network_name] = module module.network_layer_name = network_name - if hasattr(shared.sd_model, 'unet'): - for name, module in shared.sd_model.unet.named_modules(): + if hasattr(sd_model, 'unet'): + for name, module in sd_model.unet.named_modules(): network_name = "lora_unet_" + name.replace(".", "_") network_layer_mapping[network_name] = module module.network_layer_name = network_name - if hasattr(shared.sd_model, 'transformer'): - for name, module in shared.sd_model.transformer.named_modules(): + if hasattr(sd_model, 'transformer'): + for name, module in sd_model.transformer.named_modules(): network_name = "lora_transformer_" + name.replace(".", "_") network_layer_mapping[network_name] = module if "norm" in network_name and "linear" not in network_name: continue module.network_layer_name = network_name else: - if not hasattr(shared.sd_model, 'cond_stage_model'): + if not hasattr(sd_model, 'cond_stage_model'): sd_model.network_layer_mapping = {} return - for name, module in shared.sd_model.cond_stage_model.wrapped.named_modules(): + for name, module in sd_model.cond_stage_model.wrapped.named_modules(): network_name = name.replace(".", "_") network_layer_mapping[network_name] = module module.network_layer_name = network_name - for name, module in shared.sd_model.model.named_modules(): + for name, module in sd_model.model.named_modules(): network_name = name.replace(".", "_") network_layer_mapping[network_name] = module module.network_layer_name = network_name - sd_model.network_layer_mapping = network_layer_mapping + shared.sd_model.network_layer_mapping = network_layer_mapping def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_default_multiplier) -> network.Network: