diff --git a/CHANGELOG.md b/CHANGELOG.md index c819a8584..03eb91b8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,9 @@ ## Update for 2023-09-13 -Mostly a service release, but with some changes in behavior, especially in HiRes area of the code... - +Started as a mostly a service release with quite a few fixes, but then... +Major changes how **hires** works as well as support for a very interesting new model [wuerstchen](https://huggingface.co/blog/wuertschen) + - tons of fixes - changes to **hires** - enable non-latent upscale modes (standard upscalers) @@ -20,6 +21,10 @@ Mostly a service release, but with some changes in behavior, especially in HiRes - all operations are captured in image medata - diffusers: - allow loading of sd/sdxl models from safetensors without online connectivity + - support for new model: [wuerstchen](https://huggingface.co/warp-ai/wuerstchen) + its a high-resolution model (1024px+) that nearly doubls performance of sd-xl with much lower resource requirements + go to *models -> huggingface -> search "warp-ai/wuerstchen" -> download* + its nearly 12gb in size, so be patient :) - minor re-layout of the main ui - update **ui hints** - updated **models -> civitai** diff --git a/README.md b/README.md index bb5725346..bc1f07692 100644 --- a/README.md +++ b/README.md @@ -46,20 +46,23 @@ All Individual features are not listed here, instead check [ChangeLog](CHANGELOG - **Original**: Based on [LDM](https://github.com/Stability-AI/stablediffusion) reference implementation and significantly expanded on by [A1111](https://github.com/AUTOMATIC1111/stable-diffusion-webui) This is the default backend and it is fully compatible with all existing functionality and extensions + It supports **SD 1.x** and **SD 2.x** models - **Diffusers**: Based on new [Huggingface Diffusers](https://huggingface.co/docs/diffusers/index) implementation - It is also the only backend that supports **Stable Diffusion XL** model + It supports All models listed below + It is also the *only backend* that supports **Stable Diffusion XL** model See [wiki article](https://github.com/vladmandic/automatic/wiki/Diffusers) for more information ## Model support Additional models will be added as they become available and there is public interest in them -- Stable Diffusion 1.x and 2.x *(all variants)* -- Stable Diffusion XL -- Kandinsky 2.1 and 2.2 -- DeepFloyd IF -- UniDiffusion -- SD-Distilled *(all variants)* +- [Stable Diffusion](https://github.com/Stability-AI/stablediffusion/) 1.x and 2.x *(all variants)* +- [Stable Diffusion XL](https://github.com/Stability-AI/generative-models) +- [Kandinsky](https://github.com/ai-forever/Kandinsky-2) 2.1 and 2.2 +- [DeepFloyd IF](https://github.com/deep-floyd/IF) +- [UniDiffusion](https://github.com/thu-ml/unidiffuser) +- [SD-Distilled](https://huggingface.co/blog/sd_distillation) *(all variants)* +- [Wuerstchen](https://huggingface.co/blog/wuertschen) ## Platform support diff --git a/modules/modelloader.py b/modules/modelloader.py index f9b6048a0..7d99f1714 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -213,7 +213,7 @@ def load_diffusers_models(model_path: str, command_path: str = None): mtime = os.path.getmtime(folder) info = os.path.join(folder, "model_info.json") diffuser_repos.append({ 'name': name, 'filename': name, 'path': folder, 'hash': commit, 'mtime': mtime, 'model_info': info }) - if os.path.exists(os.path.join(place, folder, 'snapshots', commit, "hidden")): + if os.path.exists(os.path.join(folder, 'hidden')): continue output.append(name) except Exception as e: diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 08eac3a37..e2f5187df 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -181,7 +181,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro negative_embed = None negative_pooled = None prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2) - if shared.opts.prompt_attention in {'Compel parser', 'Full parser'}: + if shared.opts.prompt_attention in {'Compel parser', 'Full parser'} and 'StableDiffusion' in model.__class__.__name__: prompt_embed, pooled, negative_embed, negative_pooled = prompt_parser_diffusers.compel_encode_prompts(model, prompts, negative_prompts, prompts_2, negative_prompts_2, is_refiner, kwargs.pop("clip_skip", None)) if 'prompt' in possible: if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and prompt_embed is not None: diff --git a/modules/sd_models.py b/modules/sd_models.py index 5fc50dd61..e83f461c2 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -701,6 +701,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No try: shared.log.debug(f'Model load {op} config: {diffusers_load_config}') sd_model = diffusers.DiffusionPipeline.from_pretrained(model_file, **diffusers_load_config) + sd_model.model_type = sd_model.__class__.__name__ except Exception as e: shared.log.error(f'Failed loading model: {model_file} {e}') list_models() # rescan for downloaded model @@ -722,10 +723,16 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No # shared.log.info(f'Loading diffuser {op}: {checkpoint_info.filename}') if not os.path.isfile(checkpoint_info.path): try: + # os.environ.setdefault('HUGGINGFACE_HUB_CACHE', shared.opts.diffusers_dir) # evalulated only on initial diffusers load + # diffusers_load_config["cache_dir "] = shared.opts.diffusers_dir # ignored for connected pipelines such as kandinsky-prior + # diffusers.utils.constants.DIFFUSERS_CACHE = shared.opts.diffusers_dir # shared.log.debug(f'Diffusers load {op} config: {diffusers_load_config}') - sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) + # sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) + sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model.model_type = sd_model.__class__.__name__ except Exception as e: shared.log.error(f'Failed loading model {op}: {checkpoint_info.path} {e}') + return else: diffusers_load_config["local_files_only "] = True diffusers_load_config["extract_ema"] = shared.opts.diffusers_extract_ema @@ -1184,7 +1191,7 @@ def apply_token_merging(sd_model, token_merging_ratio=0): return if current_token_merging_ratio > 0: tomesd.remove_patch(sd_model) - if token_merging_ratio > 0: + if token_merging_ratio > 0 and sd_model.model_type in ['ldm', 'sd', 'sdxl']: shared.log.debug(f'Applying token merging: ratio={token_merging_ratio}') tomesd.apply_patch( sd_model, @@ -1194,4 +1201,4 @@ def apply_token_merging(sd_model, token_merging_ratio=0): merge_crossattn=False, merge_mlp=False ) - sd_model.applied_token_merged_ratio = token_merging_ratio + sd_model.applied_token_merged_ratio = token_merging_ratio diff --git a/modules/shared.py b/modules/shared.py index b5fbbe12a..01f39e13c 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -185,8 +185,9 @@ class State: image = modules.sd_samplers.samples_to_image_grid(self.current_latent) if opts.show_progress_grid else modules.sd_samplers.sample_to_image(self.current_latent) self.assign_current_image(image) self.current_image_sampling_step = self.sampling_step - except Exception as e: - log.error(f'Error setting current image: step={self.sampling_step} {e}') + except Exception: + # log.error(f'Error setting current image: step={self.sampling_step} {e}') + pass def assign_current_image(self, image): self.current_image = image diff --git a/requirements.txt b/requirements.txt index 41e37dfa1..67aad81ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,13 +44,13 @@ fasteners typing-extensions==4.7.1 antlr4-python3-runtime==4.9.3 requests==2.31.0 -tqdm==4.65.0 +tqdm==4.66.1 accelerate==0.20.3 opencv-python-headless==4.7.0.72 diffusers==0.21.1 einops==0.4.1 gradio==3.43.2 -huggingface_hub==0.16.4 +huggingface_hub==0.17.1 numexpr==2.8.4 numpy==1.24.4 numba==0.57.1