diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ea1430f5..540bc0acc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Upgrades are still possible and supported, but above is recommended for best exp - refactored subfolder handling *note*: this will trigger model hash recaclulation on first model use - **Diffusers**: + - better pipeline auto-detect when loading from safetensors - **SDXL Inpaint** - Although any model can be used for inpainiting, there is a case to be made for dedicated inpainting models as they are tuned to inpaint and not generate @@ -39,6 +40,8 @@ Upgrades are still possible and supported, but above is recommended for best exp To download go to *Models -> Huggingface*: - `stabilityai/sd-x2-latent-upscaler` *(2.2GB)* - `stabilityai/stable-diffusion-x4-upscaler` *(1.7GB)* + - better **Embeddings** support for SD and SDXL + faster loading, wider compatibility and support for embeddings with multiple vectors - **Upscalers**: - more high quality upscalers available by default *SwinIR:2, ESRGAN:12, RealESRGAN:6, SCUNet:2* @@ -62,10 +65,11 @@ Upgrades are still possible and supported, but above is recommended for best exp - CivitAI integration in *Models -> CivitAI* can now find most previews AND metadata for most models (checkpoints, loras, embeddings) metadata is now parsed and saved in *[model].json* + typical hit rate is >95% for models, loras and embeddings - Description from parsed model metadata is used as model description if there is no manual description file present in format of *[model].txt* -- **Diffusers** - - better pipeline auto-detect when loading from safetensors + - to enable search, make sure all models have set hash values + *Models -> Valida -> Calculate hashes* - **Startup** - All main CLI parameters can now be set as environment variable as well for example `--data-dir ` can be specified as `SD_DATADIR=` before starting SD.Next diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 6b70e0200..0f10f968b 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -177,8 +177,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro return prompts, negative_prompts, prompts_2, negative_prompts_2 def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs): - if hasattr(model, 'embedding_db'): - del model.embedding_db + # if hasattr(model, 'embedding_db'): + # del model.embedding_db try: is_refiner = model.text_encoder.__class__.__name__ != 'CLIPTextModel' except Exception: diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 4e1c8da05..00bbec750 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -62,8 +62,8 @@ class DiffusersTextualInversionManager(BaseTextualInversionManager): replacement += f" {token}_{i}" i += 1 prompt = prompt.replace(token, replacement) - if hasattr(self.pipe, 'embedding_db'): - self.pipe.embedding_db.embeddings_used = list(set(self.pipe.embedding_db.embeddings_used)) + if hasattr(self.pipe, 'embedding_db'): + self.pipe.embedding_db.embeddings_used = list(set(self.pipe.embedding_db.embeddings_used)) return prompt def expand_textual_inversion_token_ids_if_necessary(self, token_ids: typing.List[int]) -> typing.List[int]: diff --git a/modules/sd_models.py b/modules/sd_models.py index 72dc883a7..443a09167 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -249,7 +249,7 @@ def select_checkpoint(op='model'): return None checkpoint_info = get_closet_checkpoint_match(model_checkpoint) if checkpoint_info is not None: - shared.log.info(f'Select model: {op}="{checkpoint_info.title if checkpoint_info is not None else None}"') + shared.log.info(f'Select: {op}="{checkpoint_info.title if checkpoint_info is not None else None}"') return checkpoint_info if len(checkpoints_list) == 0 and not shared.cmd_opts.no_download: shared.log.error("Cannot run without a checkpoint") @@ -263,7 +263,7 @@ def select_checkpoint(op='model'): shared.log.info("Selecting first available checkpoint") # shared.log.warning(f"Loading fallback checkpoint: {checkpoint_info.title}") shared.opts.data['sd_model_checkpoint'] = checkpoint_info.title - shared.log.info(f'Select model: {op}="{checkpoint_info.title if checkpoint_info is not None else None}"') + shared.log.info(f'Select: {op}="{checkpoint_info.title if checkpoint_info is not None else None}"') return checkpoint_info @@ -946,10 +946,9 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.embedding_db.load_textual_inversion_embeddings(force_reload=True) timer.record("load") - shared.log.info(f"Model loaded in {timer.summary()} native={get_native(sd_model)}") devices.torch_gc(force=True) script_callbacks.model_loaded_callback(sd_model) - shared.log.info(f'Model load finished {op}: {memory_stats()}') + shared.log.info(f"Loaded {op}: time={timer.summary()} native={get_native(sd_model)} {memory_stats()}") class DiffusersTaskType(Enum): @@ -958,11 +957,13 @@ class DiffusersTaskType(Enum): INPAINTING = 3 INSTRUCT = 4 + def set_diffuser_pipe(pipe, new_pipe_type): sd_checkpoint_info = getattr(pipe, "sd_checkpoint_info", None) sd_model_checkpoint = getattr(pipe, "sd_model_checkpoint", None) sd_model_hash = getattr(pipe, "sd_model_hash", None) has_accelerate = getattr(pipe, "has_accelerate", None) + embedding_db = getattr(pipe, "embedding_db", None) if new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE and (pipe.__class__.__name__ == "StableDiffusionXLPipeline" or pipe.__class__.__name__ == 'StableDiffusionXLImg2ImgPipeline'): new_pipe_type = DiffusersTaskType.INPAINTING # sdxl works better with init mask @@ -984,6 +985,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): new_pipe.sd_model_checkpoint = sd_model_checkpoint new_pipe.sd_model_hash = sd_model_hash new_pipe.has_accelerate = has_accelerate + new_pipe.embedding_db = embedding_db model_data.sd_model = new_pipe shared.log.debug(f"Pipeline class changed from {pipe.__class__.__name__} to {new_pipe.__class__.__name__}") diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index eec6f6bd2..ff42659e7 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -1,6 +1,7 @@ import os import html import csv +import time from collections import namedtuple import torch from tqdm import tqdm @@ -134,25 +135,39 @@ class EmbeddingDatabase: name = os.path.basename(fn) embedding = Embedding(vec=None, name=name, filename=path) try: + done = False if hasattr(pipe,"load_textual_inversion"): - pipe.load_textual_inversion(path, cache_dir=shared.opts.diffusers_dir, local_files_only=True) - elif "safetensors" in path: + try: + pipe.load_textual_inversion(path, cache_dir=shared.opts.diffusers_dir, local_files_only=True) + done = True + except Exception: + pass + if not done and "safetensors" in path: embeddings_dict = {} from safetensors.torch import safe_open with safe_open(path, framework="pt") as f: for k in f.keys(): embeddings_dict[k] = f.get_tensor(k) + tokens = [] for i in range(len(embeddings_dict["clip_l"])): - if i == 0: - token = name.lower() - else: - token = f"{name.lower()}_{i}" - pipe.tokenizer.add_tokens(token) - token_id = pipe.tokenizer.convert_tokens_to_ids(token) - pipe.text_encoder.resize_token_embeddings(len(pipe.tokenizer)) - pipe.text_encoder_2.resize_token_embeddings(len(pipe.tokenizer)) - pipe.text_encoder.get_input_embeddings().weight.data[token_id] = embeddings_dict["clip_l"][i] - pipe.text_encoder_2.get_input_embeddings().weight.data[token_id] = embeddings_dict["clip_g"][i] + tokens.append(name if i == 0 else f"{name}_{i}") + num_added = pipe.tokenizer.add_tokens(tokens) + if num_added > 0: + token_ids = pipe.tokenizer.convert_tokens_to_ids(tokens) + clip_l = None + clip_g = None + if hasattr(pipe.text_encoder, "resize_token_embeddings"): + pipe.text_encoder.resize_token_embeddings(len(pipe.tokenizer)) + clip_l = pipe.text_encoder.get_input_embeddings().weight + if hasattr(pipe.text_encoder_2, "resize_token_embeddings"): + pipe.text_encoder_2.resize_token_embeddings(len(pipe.tokenizer)) + clip_g = pipe.text_encoder_2.get_input_embeddings().weight + for i in range(len(token_ids)): + if clip_l is not None: + clip_l.data[token_ids[i]] = embeddings_dict["clip_l"][i] + if clip_g is not None: + clip_g.data[token_ids[i]] = embeddings_dict["clip_g"][i] + else: raise NotImplementedError # self.word_embeddings[name] = embedding @@ -234,6 +249,7 @@ class EmbeddingDatabase: continue def load_textual_inversion_embeddings(self, force_reload=False): + t0 = time.time() if not force_reload: need_reload = False for embdir in self.embedding_dirs.values(): @@ -260,7 +276,8 @@ class EmbeddingDatabase: displayed_embeddings = (tuple(self.word_embeddings.keys()), tuple(self.skipped_embeddings.keys())) if self.previously_displayed_embeddings != displayed_embeddings: self.previously_displayed_embeddings = displayed_embeddings - shared.log.info(f"Loaded embeddings: loaded={len(self.word_embeddings)} skipped={len(self.skipped_embeddings)}") + t1 = time.time() + shared.log.info(f"Loaded embeddings: loaded={len(self.word_embeddings)} skipped={len(self.skipped_embeddings)} time={t1-t0:.2f}s") def find_embedding_at_position(self, tokens, offset):