From 1502dc2e13636953e9473f8db4db3a2564e763ef Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 25 Nov 2023 13:27:16 -0500 Subject: [PATCH] add kandinsky3 support --- CHANGELOG.md | 8 +++++++- extensions-builtin/Lora/extra_networks_lora.py | 5 +++-- html/reference.json | 5 +++++ modules/shared_items.py | 2 +- scripts/ipadapter.py | 12 ++++++------ 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad98ed3db..48f70e970 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,13 +12,19 @@ Note: Release pending `diffusers==0.24` - In *Advanced* params - Allows control of *latent clamping*, *color centering* and *range maximimization* - Supported by *XYZ grid* + - **Kandinsky 3** support + - download using built-in model downloader or simply select from networks -> reference + - this model is absolutely massive at 27.5GB at fp16, so be patient + - model params count is at 11.9B (compared to SD-XL at 3.3B) and its trained on mixed resolutions from 256px to 1024px + - use either model offload or sequential cpu offload to be able to use it - better autodetection of *inpaint* and *instruct* pipelines - support long seconary prompt for refiner - **General** - log level defaults to info for console and debug for log file - better prompt display in process tab - increase maximum lora cache values - - fix for python 3.9 compatibility + - fix controlnet compatibility issues in original backend + - fix python 3.9 compatibility issues - fix img2img/inpaint paste params ## Update for 2023-11-23 diff --git a/extensions-builtin/Lora/extra_networks_lora.py b/extensions-builtin/Lora/extra_networks_lora.py index 82aa82454..5256ea94a 100644 --- a/extensions-builtin/Lora/extra_networks_lora.py +++ b/extensions-builtin/Lora/extra_networks_lora.py @@ -64,8 +64,9 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): self.active = False def deactivate(self, p): - if shared.backend == shared.Backend.DIFFUSERS and hasattr(shared.sd_model, "unload_lora_weights"): - shared.sd_model.unload_lora_weights() + if shared.backend == shared.Backend.DIFFUSERS and hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"): + if 'CLIP' in shared.sd_model.text_encoder.__class__.__name__: + shared.sd_model.unload_lora_weights() if not self.active and getattr(networks, "originals", None ) is not None: networks.originals.undo() # remove patches if networks.debug: diff --git a/html/reference.json b/html/reference.json index d1b2884dc..5ea827fe1 100644 --- a/html/reference.json +++ b/html/reference.json @@ -54,6 +54,11 @@ "desc": "Kandinsky 2.2 is a text-conditional diffusion model (+0.1!) based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder. Kandinsky 2.1 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas. It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.", "preview": "kandinsky-community--kandinsky-2-2-decoder.jpg" }, + "Kandinsky 3": { + "path": "kandinsky-community/kandinsky-3", + "desc": "Kandinsky 3.0 is an open-source text-to-image diffusion model built upon the Kandinsky2-x model family. In comparison to its predecessors, Kandinsky 3.0 incorporates more data and specifically related to Russian culture, which allows to generate pictures related to Russin culture. Furthermore, enhancements have been made to the text understanding and visual quality of the model, achieved by increasing the size of the text encoder and Diffusion U-Net models, respectively.", + "preview": "kandinsky-community--kandinsky-3.jpg" + }, "DeepFloyd IF Medium": { "path": "DeepFloyd/IF-I-M-v1.0", "desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.", diff --git a/modules/shared_items.py b/modules/shared_items.py index f378b93e5..b0d4f1ec4 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -44,7 +44,7 @@ def get_pipelines(): 'Wuerstchen': getattr(diffusers, 'WuerstchenCombinedPipeline', None), 'Kandinsky 2.1': getattr(diffusers, 'KandinskyPipeline', None), 'Kandinsky 2.2': getattr(diffusers, 'KandinskyV22Pipeline', None), - # 'Kandinsky 3': getattr(diffusers, 'KandinskyV3Pipeline', None), + 'Kandinsky 3': getattr(diffusers, 'Kandinsky3Pipeline', None), 'DeepFloyd IF': getattr(diffusers, 'IFPipeline', None), 'Custom Diffusers Pipeline': getattr(diffusers, 'DiffusionPipeline', None), # Segmind SSD-1B, Segmind Tiny diff --git a/scripts/ipadapter.py b/scripts/ipadapter.py index 1bc7a3a7a..33e5f623a 100644 --- a/scripts/ipadapter.py +++ b/scripts/ipadapter.py @@ -58,12 +58,6 @@ class Script(scripts.Script): if shared.backend != shared.Backend.DIFFUSERS: shared.log.warning('IP adapter: not in diffusers mode') return - if not hasattr(shared.sd_model, 'load_ip_adapter'): - shared.log.error(f'IP adapter: pipeline not supported: {shared.sd_model.__class__.__name__}') - return - if image is None: - shared.log.error('IP adapter: no image') - return if adapter == 'none': if hasattr(shared.sd_model, 'set_ip_adapter_scale'): shared.sd_model.set_ip_adapter_scale(0) @@ -73,6 +67,12 @@ class Script(scripts.Script): shared.sd_model.unet.config.encoder_hid_dim_type = None loaded = None return + if image is None: + shared.log.error('IP adapter: no image') + return + if not hasattr(shared.sd_model, 'load_ip_adapter'): + shared.log.error(f'IP adapter: pipeline not supported: {shared.sd_model.__class__.__name__}') + return if getattr(shared.sd_model, 'image_encoder', None) is None: if shared.sd_model_type == 'sd': subfolder = 'models/image_encoder'