add kandinsky3 support

CHANGELOG.md

@@ -12,13 +12,19 @@ Note: Release pending `diffusers==0.24`
     - In *Advanced* params
     - Allows control of *latent clamping*, *color centering* and *range maximimization*
     - Supported by *XYZ grid*
+  - **Kandinsky 3** support  
+    - download using built-in model downloader or simply select from networks -> reference  
+    - this model is absolutely massive at 27.5GB at fp16, so be patient
+    - model params count is at 11.9B (compared to SD-XL at 3.3B) and its trained on mixed resolutions from 256px to 1024px  
+    - use either model offload or sequential cpu offload to be able to use it
   - better autodetection of *inpaint* and *instruct* pipelines
   - support long seconary prompt for refiner
 - **General**
   - log level defaults to info for console and debug for log file
   - better prompt display in process tab
   - increase maximum lora cache values
-  - fix for python 3.9 compatibility
+  - fix controlnet compatibility issues in original backend
+  - fix python 3.9 compatibility issues
   - fix img2img/inpaint paste params
 
 ## Update for 2023-11-23

extensions-builtin/Lora/extra_networks_lora.py

@@ -64,8 +64,9 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
             self.active = False
 
     def deactivate(self, p):
-        if shared.backend == shared.Backend.DIFFUSERS and hasattr(shared.sd_model, "unload_lora_weights"):
-            shared.sd_model.unload_lora_weights()
+        if shared.backend == shared.Backend.DIFFUSERS and hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"):
+            if 'CLIP' in shared.sd_model.text_encoder.__class__.__name__:
+                shared.sd_model.unload_lora_weights()
         if not self.active and getattr(networks, "originals", None ) is not None:
             networks.originals.undo() # remove patches
             if networks.debug:

html/reference.json

@@ -54,6 +54,11 @@
     "desc": "Kandinsky 2.2 is a text-conditional diffusion model (+0.1!) based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder. Kandinsky 2.1 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas. It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.",
     "preview": "kandinsky-community--kandinsky-2-2-decoder.jpg"
   },
+  "Kandinsky 3": {
+    "path": "kandinsky-community/kandinsky-3",
+    "desc": "Kandinsky 3.0 is an open-source text-to-image diffusion model built upon the Kandinsky2-x model family. In comparison to its predecessors, Kandinsky 3.0 incorporates more data and specifically related to Russian culture, which allows to generate pictures related to Russin culture. Furthermore, enhancements have been made to the text understanding and visual quality of the model, achieved by increasing the size of the text encoder and Diffusion U-Net models, respectively.",
+    "preview": "kandinsky-community--kandinsky-3.jpg"
+  },
   "DeepFloyd IF Medium": {
     "path": "DeepFloyd/IF-I-M-v1.0",
     "desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.",

modules/shared_items.py

@@ -44,7 +44,7 @@ def get_pipelines():
         'Wuerstchen': getattr(diffusers, 'WuerstchenCombinedPipeline', None),
         'Kandinsky 2.1': getattr(diffusers, 'KandinskyPipeline', None),
         'Kandinsky 2.2': getattr(diffusers, 'KandinskyV22Pipeline', None),
-        # 'Kandinsky 3': getattr(diffusers, 'KandinskyV3Pipeline', None),
+        'Kandinsky 3': getattr(diffusers, 'Kandinsky3Pipeline', None),
         'DeepFloyd IF': getattr(diffusers, 'IFPipeline', None),
         'Custom Diffusers Pipeline': getattr(diffusers, 'DiffusionPipeline', None),
         # Segmind SSD-1B, Segmind Tiny

scripts/ipadapter.py

@@ -58,12 +58,6 @@ class Script(scripts.Script):
         if shared.backend != shared.Backend.DIFFUSERS:
             shared.log.warning('IP adapter: not in diffusers mode')
             return
-        if not hasattr(shared.sd_model, 'load_ip_adapter'):
-            shared.log.error(f'IP adapter: pipeline not supported: {shared.sd_model.__class__.__name__}')
-            return
-        if image is None:
-            shared.log.error('IP adapter: no image')
-            return
         if adapter == 'none':
             if hasattr(shared.sd_model, 'set_ip_adapter_scale'):
                 shared.sd_model.set_ip_adapter_scale(0)
@@ -73,6 +67,12 @@ class Script(scripts.Script):
                 shared.sd_model.unet.config.encoder_hid_dim_type = None
                 loaded = None
             return
+        if image is None:
+            shared.log.error('IP adapter: no image')
+            return
+        if not hasattr(shared.sd_model, 'load_ip_adapter'):
+            shared.log.error(f'IP adapter: pipeline not supported: {shared.sd_model.__class__.__name__}')
+            return
         if getattr(shared.sd_model, 'image_encoder', None) is None:
             if shared.sd_model_type == 'sd':
                 subfolder = 'models/image_encoder'