diff --git a/CHANGELOG.md b/CHANGELOG.md index 50d1ca0d8..4d81c6fa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,6 @@ ## Update for 2023-12-27 -*Note*: based on `diffusers==0.25.0.dev0` - - **Control** - native implementation of all image control methods: **ControlNet**, **ControlNet XS**, **Control LLLite**, **T2I Adapters** and **IP Adapters** @@ -13,6 +11,25 @@ - for details and list of supported models and workflows, see Wiki documentation: - **Diffusers** + - [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega) model support + - small and fast version of **SDXL**, only 3.1GB in size! + - select from *networks -> reference* + - [aMUSEd 256](https://huggingface.co/amused/amused-256) and [aMUSEd 512](https://huggingface.co/amused/amused-512) model support + - lightweigt models that excel at fast image generation + - *note*: must select: settings -> diffusers -> generator device: unset + - select from *networks -> reference* + - [Playground v1](https://huggingface.co/playgroundai/playground-v1), [Playground v2 256](https://huggingface.co/playgroundai/playground-v2-256px-base), [Playground v2 512](https://huggingface.co/playgroundai/playground-v2-512px-base), [Playground v2 1024](https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic) model support + - comparable to SD15 and SD-XL, trained from scratch for highly aesthetic images + - simply select from *networks -> reference* and use as usual + - [BLIP-Diffusion](https://dxli94.github.io/BLIP-Diffusion-website/) + - img2img model that can replace subjects in images using prompt keywords + - download and load by selecting from *networks -> reference -> blip diffusion* + - in image tab, select `blip diffusion` script + - [DemoFusion](https://github.com/PRIS-CV/DemoFusion) run your SDXL generations at any resolution! + - in **Text** tab select *script* -> *demofusion* + - *note*: GPU VRAM limits do not automatically go away so be careful when using it with large resolutions + in the future, expect more optimizations, especially related to offloading/slicing/tiling, + but at the moment this is pretty much experimental-only - **AnimateDiff** - can now be used with *second pass* - enhance, upscale and hires your videos! - **IP Adapter** @@ -30,20 +47,6 @@ *if you know of any other t2v models you'd like to see supported, let me know!* - models are auto-downloaded on first use - *note*: current base model will be unloaded to free up resources - - [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega) support - - small and fast version of **SDXL**, only 3.1GB in size! - - select from *networks -> reference* - - [Playground v1](https://huggingface.co/playgroundai/playground-v1), [Playground v2 256](https://huggingface.co/playgroundai/playground-v2-256px-base), [Playground v2 512](https://huggingface.co/playgroundai/playground-v2-512px-base), [Playground v2 1024](https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic) model support - - simply select from *networks -> reference* and use as usual - - [BLIP-Diffusion](https://dxli94.github.io/BLIP-Diffusion-website/) - - img2img model that can replace subjects in images using prompt keywords - - download and load by selecting from *networks -> reference -> blip diffusion* - - in image tab, select `blip diffusion` script - - [DemoFusion](https://github.com/PRIS-CV/DemoFusion) run your SDXL generations at any resolution! - - in **Text** tab select *script* -> *demofusion* - - *note*: GPU VRAM limits do not automatically go away so be careful when using it with large resolutions - in the future, expect more optimizations, especially related to offloading/slicing/tiling, - but at the moment this is pretty much experimental-only - **Prompt scheduling** now implemented for Diffusers backend, thanks @AI-Casanova - **Custom pipelines** contribute by adding your own custom pipelines! - for details, see fully documented example: diff --git a/html/reference.json b/html/reference.json index 6ad5f0700..1ab1291f7 100644 --- a/html/reference.json +++ b/html/reference.json @@ -144,6 +144,16 @@ "desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.", "preview": "DeepFloyd--IF-I-M-v1.0.jpg" }, + "aMUSEd 256": { + "path": "amused/amused-256", + "desc": "Amused is a lightweight text to image model based off of the muse architecture. Amused is particularly useful in applications that require a lightweight and fast model such as generating many images quickly at once.", + "preview": "amused--amused-256.jpg" + }, + "aMUSEd 512": { + "path": "amused/amused-512", + "desc": "Amused is a lightweight text to image model based off of the muse architecture. Amused is particularly useful in applications that require a lightweight and fast model such as generating many images quickly at once.", + "preview": "amused--amused-512.jpg" + }, "Tsinghua UniDiffuser": { "path": "thu-ml/unidiffuser-v1", "desc": "UniDiffuser is a unified diffusion framework to fit all distributions relevant to a set of multi-modal data in one transformer. UniDiffuser is able to perform image, text, text-to-image, image-to-text, and image-text pair generation by setting proper timesteps without additional overhead.\nSpecifically, UniDiffuser employs a variation of transformer, called U-ViT, which parameterizes the joint noise prediction network. Other components perform as encoders and decoders of different modalities, including a pretrained image autoencoder from Stable Diffusion, a pretrained image ViT-B/32 CLIP encoder, a pretrained text ViT-L CLIP encoder, and a GPT-2 text decoder finetuned by ourselves.", diff --git a/models/Reference/amused--amused-256.jpg b/models/Reference/amused--amused-256.jpg new file mode 100644 index 000000000..f410817a8 Binary files /dev/null and b/models/Reference/amused--amused-256.jpg differ diff --git a/models/Reference/amused--amused-512.jpg b/models/Reference/amused--amused-512.jpg new file mode 100644 index 000000000..0b8e26240 Binary files /dev/null and b/models/Reference/amused--amused-512.jpg differ diff --git a/modules/dml/__init__.py b/modules/dml/__init__.py index 3b9c8cf63..358885650 100644 --- a/modules/dml/__init__.py +++ b/modules/dml/__init__.py @@ -85,7 +85,7 @@ class OverrideItem(NamedTuple): message: Optional[str] opts_override_table = { - "diffusers_generator_device": OverrideItem("cpu", None, "DirectML does not support torch Generator API"), + "diffusers_generator_device": OverrideItem("CPU", None, "DirectML does not support torch Generator API"), "diffusers_model_cpu_offload": OverrideItem(False, None, "Diffusers model CPU offloading does not support DirectML devices"), "diffusers_seq_cpu_offload": OverrideItem(False, lambda opts: opts.diffusers_pipeline != "Stable Diffusion XL", "Diffusers sequential CPU offloading is available only on StableDiffusionXLPipeline with DirectML devices"), } diff --git a/modules/postprocess/sdupscaler_model.py b/modules/postprocess/sdupscaler_model.py index f4c069925..0d73b8fa9 100644 --- a/modules/postprocess/sdupscaler_model.py +++ b/modules/postprocess/sdupscaler_model.py @@ -45,7 +45,7 @@ class UpscalerSD(Upscaler): if model is None: return img seeds = [torch.randint(0, 2 ** 32, (1,)).item() for _ in range(1)] - generator_device = devices.cpu if shared.opts.diffusers_generator_device == "cpu" else devices.device + generator_device = devices.cpu if shared.opts.diffusers_generator_device == "CPU" else devices.device generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds] args = { 'prompt': '', diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 702a8e730..2d881a1b1 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -213,8 +213,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro signature = inspect.signature(type(model).__call__) possible = signature.parameters.keys() debug(f'Diffusers pipeline possible: {possible}') - generator_device = devices.cpu if shared.opts.diffusers_generator_device == "cpu" else shared.device - generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds] + if shared.opts.diffusers_generator_device == "Unset": + generator_device = None + generator = None + else: + generator_device = devices.cpu if shared.opts.diffusers_generator_device == "CPU" else shared.device + generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds] prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2) parser = 'Fixed attention' if shared.opts.prompt_attention != 'Fixed attention' and 'StableDiffusion' in model.__class__.__name__: @@ -247,7 +251,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro args['noise_sampler_seed'] = seeds[0] if 'guidance_scale' in possible: args['guidance_scale'] = p.cfg_scale - if 'generator' in possible: + if 'generator' in possible and generator is not None: args['generator'] = generator if 'output_type' in possible: args['output_type'] = 'np' @@ -363,6 +367,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro def update_sampler(sd_model, second_pass=False): sampler_selection = p.latent_sampler if second_pass else p.sampler_name # is_karras_compatible = sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers + if sd_model.__class__.__name__ in ['AmusedPipeline']: + return # models with their own schedulers if hasattr(sd_model, 'scheduler') and sampler_selection != 'Default': sampler = sd_samplers.all_samplers_map.get(sampler_selection, None) if sampler is None: diff --git a/modules/sd_models.py b/modules/sd_models.py index 85f8894d4..60d4244ef 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -756,6 +756,8 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model'): devices.dtype_vae = torch.float32 sd_model.vae.to(devices.dtype_vae) shared.log.debug(f'Setting {op} VAE: name={sd_vae.loaded_vae_file} upcast={sd_model.vae.config.get("force_upcast", None)}') + if hasattr(sd_model, "vqvae"): + sd_model.vqvae.to(torch.float32) # vqvae is producing nans in fp16 if shared.opts.cross_attention_optimization == "xFormers" and hasattr(sd_model, 'enable_xformers_memory_efficient_attention'): sd_model.enable_xformers_memory_efficient_attention() diff --git a/modules/shared.py b/modules/shared.py index cd70bf54d..150f0af8d 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -373,7 +373,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), { "diffusers_move_unet": OptionInfo(True, "Move base model to CPU when using VAE"), "diffusers_move_refiner": OptionInfo(True, "Move refiner model to CPU when not in use"), "diffusers_extract_ema": OptionInfo(True, "Use model EMA weights when possible"), - "diffusers_generator_device": OptionInfo("default", "Generator device", gr.Radio, {"choices": ["default", "cpu"]}), + "diffusers_generator_device": OptionInfo("GPU", "Generator device", gr.Radio, {"choices": ["GPU", "CPU", "Unset"]}), "diffusers_model_cpu_offload": OptionInfo(False, "Enable model CPU offload (--medvram)"), "diffusers_seq_cpu_offload": OptionInfo(False, "Enable sequential CPU offload (--lowvram)"), "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, {"choices": ['default', 'true', 'false']}), diff --git a/requirements.txt b/requirements.txt index ce01e21c4..ab5c24b24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,7 +51,7 @@ requests==2.31.0 tqdm==4.66.1 accelerate==0.25.0 opencv-contrib-python-headless==4.8.1.78 -diffusers==0.24.0 +diffusers==0.25.0 einops==0.4.1 gradio==3.43.2 huggingface_hub==0.20.1