update diffusers

CHANGELOG.md

@@ -1,8 +1,22 @@
 # Change Log for SD.Next
 
-## Update for 07/11/2023:
+## Update for 07/12/2023
+
+Another big one...
+
+- diffusers backend:
+  - separate ui settings for refiner pass with sd-xl  
+    you can specify: prompt, negative prompt, steps, denoise start  
+  - fix loading from pure safetensors files  
+    now you can load sd-xl from safetensors file or from huggingface folder format  
+  - fix kandinsky model  
+- other:
+  - major refactoring of the javascript code  
+    includes fixes for text selections and navigation  
+  - minor fixes in extra-networks
+
+big thanks to @huggingface team for great communication, support and fixing all the reported issues asap!
 
-- fixes in extra-networks, diffusers samplers
 
 ## Update for 07/10/2023
 

TODO.md

@@ -2,12 +2,19 @@
 
 ## Issues
 
-Stuff to be fixed...
+Stuff to be fixed, in no particular order...
 
+- SD-XL Lora
+- SD-XL Img2Img/Inpaint
+- Kandinsky 2.2 (2.1 is working)
+- Refresh `sd_checkpoint` pulldown on backend switch
+- Extensions misteriously auto-enabling
+- VAE/UNet dtypes
+- Attention head
 
 ## Features
 
-Stuff to be added...
+Stuff to be added, in no particular order...
 
 - Update `Wiki`
 - Create new `GitHub` hooks/actions for CI/CD  
@@ -15,8 +22,19 @@ Stuff to be added...
 - Update `train.py` to use `interrogator`
 - Update `train.py` to use `rembg`
 - Create new train UI
-- Create new Models UI
-- Intelligent preview mode
+- Docker PR
+- Port `p.all_hr_prompts`
+- Image watermark using `image-watermark`
+- Image phash and hdash using `imagehash`
+- Model merge using `git-rebasin`
+- Additional upscalers
+- New image browser
+- Update `gradio`
+- Rename repo: **automatic** -> **sdnext**
+- New icons
+- Enable refiner workflow for `ldm` backend
+- Improve `lyco` logging
+- Cache models when switching backends
 
 ## Investigate
 
@@ -49,19 +67,4 @@ Tech that can be integrated as part of the core workflow...
 
 - Bunch of stuff: <https://pharmapsychotic.com/tools.html>
 - <https://towardsdatascience.com/mastering-memoization-in-python-dcdd8b435189>
-- docker
-- port `p.all_hr_prompts`
-- test `lyco_patch_lora`
-- fix `lyco` logging
-- image watermark
-- image `imagehash` phash and hdash
-- git-rebasin
-- additional upscalers
-- new image browser
-- `git submodule set-url extensions-builtin/clip-interrogator-ext https://github.com/Dahvikiin/clip-interrogator-ext.git`
-- upate `gradio`
-- extra network refresh breaks if new extra network type found
-- [sd-xl lora](https://civitai.com/models/104913/fcstyledxl)
-- sd-xl img2img with configurable steps
-- change backend pipeline on-the-fly
-- rename repo
+

modules/processing.py

@@ -3,6 +3,7 @@ import math
 import os
 import hashlib
 import random
+import inspect
 from contextlib import nullcontext
 from typing import Any, Dict, List
 import torch
@@ -612,6 +613,45 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
         cache[0] = (required_prompts, steps)
         return cache[1]
 
+    # TODO Diffusers limited callbacks
+    def diffusers_callback(step: int, _timestep: int, latents: torch.FloatTensor):
+        shared.state.sampling_step = step
+        shared.state.sampling_steps = p.steps
+        shared.state.current_latent = latents
+        shared.state.set_current_image()
+
+    def set_pipeline_args(model, prompt, negative_prompt, **kwargs):
+        args = {}
+        pipeline = model.main if model.__class__.__name__ == 'PriorPipeline' else model
+        signature = inspect.signature(type(pipeline).__call__)
+        possible = signature.parameters.keys()
+        generator_device = 'cpu' if shared.opts.diffusers_generator_device == "cpu" else shared.device
+        generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds]
+        if 'prompt' in possible:
+            args['prompt'] = prompt
+        if 'negative_prompt' in possible:
+            args['negative_prompt'] = negative_prompt
+        if 'num_inference_steps' in possible:
+            args['num_inference_steps'] = p.steps
+        if 'guidance_scale' in possible:
+            args['guidance_scale'] = p.cfg_scale
+        if 'generator' in possible:
+            args['generator'] = generator
+        if 'output_type' in possible:
+            args['output_type'] = 'np'
+        if 'callback_steps' in possible:
+            args['callback_steps'] = 1
+        if 'callback' in args:
+            args['callback'] = diffusers_callback
+        if 'cross_attention_kwargs' in possible:
+            args['cross_attention_kwargs'] = cross_attention_kwargs
+        for arg in kwargs:
+            if arg in possible:
+                args[arg] = kwargs[arg]
+        log.debug(f'Diffuser pipeline: {pipeline.__class__.__name__} args={args.keys()}')
+        return args
+
+
     ema_scope_context = p.sd_model.ema_scope if shared.backend == Backend.ORIGINAL else nullcontext
     with torch.no_grad(), ema_scope_context():
         with devices.autocast():
@@ -682,8 +722,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                 del samples_ddim
 
             elif shared.backend == Backend.DIFFUSERS:
-                generator_device = 'cpu' if shared.opts.diffusers_generator_device == "cpu" else shared.device
-                generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds]
                 if (not hasattr(shared.sd_model.scheduler, 'name')) or (shared.sd_model.scheduler.name != p.sampler_name):
                     sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None)
                     if sampler is None:
@@ -702,41 +740,33 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                     # TODO(PVP): change out to latents once possible with `diffusers`
                     task_specific_kwargs = {"image": p.init_images[0], "mask_image": p.image_mask, "strength": p.denoising_strength}
 
-                # TODO Diffusers limited callbacks
                 # TODO Diffusers processing is not using p.sample so second pass is ignored
-                def diffusers_callback(step: int, _timestep: int, latents: torch.FloatTensor):
-                    shared.state.sampling_step = step
-                    shared.state.sampling_steps = p.steps
-                    shared.state.current_latent = latents
-                    shared.state.set_current_image()
 
                 shared.sd_model.to(devices.device)
-
-                pipe_args = { # TODO needs dynamic discovery of possible args
-                    "prompt": prompts,
-                    "negative_prompt": negative_prompts,
-                    "num_inference_steps": p.steps,
-                    "guidance_scale": p.cfg_scale,
-                    "generator": generator,
-                    "output_type": 'np' if shared.sd_refiner is None else 'latent',
-                    "callback_steps": 1, # TODO not supported by Kandinsky
-                    "callback": diffusers_callback, # TODO not supported by Kandinsky
-                    "cross_attention_kwargs": cross_attention_kwargs, # TODO not supported by Kandinsky
-                }
-                output = shared.sd_model(**pipe_args, **task_specific_kwargs) # pylint: disable=not-callable
-
+                pipe_args = set_pipeline_args(
+                    model=shared.sd_model,
+                    prompt=prompts,
+                    negative_prompt=negative_prompts,
+                    output_type='np' if shared.sd_refiner is None else 'latent',
+                    **task_specific_kwargs
+                )
+                output = shared.sd_model(**pipe_args) # pylint: disable=not-callable
                 if shared.sd_refiner is not None:
                     if shared.opts.diffusers_move_base:
                         shared.log.debug('Moving base model to CPU')
                         shared.sd_model.to('cpu')
                     shared.sd_refiner.to(devices.device)
                     devices.torch_gc()
-                    init_image = output.images[0]
-                    pipe_args['image'] = init_image
-                    pipe_args['output_type'] = 'np'
+                    pipe_args = set_pipeline_args(
+                        model=shared.sd_refiner,
+                        prompt=p.refiner_prompt if len(p.refiner_prompt) > 0 else prompts,
+                        negative_prompt=p.refiner_negative if len(p.refiner_negative) > 0 else negative_prompts,
+                        image=output.images[0],
+                        output_type='np'
+                    )
                     output = shared.sd_refiner(**pipe_args) # pylint: disable=not-callable
                     if shared.opts.diffusers_move_refiner:
-                        shared.log.debug('Moving refiner model to CPU')
+                        log.debug('Moving refiner model to CPU')
                         shared.sd_refiner.to('cpu')
 
                 x_samples_ddim = output.images
@@ -852,7 +882,8 @@ def old_hires_fix_first_pass_dimensions(width, height):
 class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
     sampler = None
 
-    def __init__(self, enable_hr: bool = False, denoising_strength: float = 0.75, firstphase_width: int = 0, firstphase_height: int = 0, hr_scale: float = 2.0, hr_upscaler: str = None, hr_second_pass_steps: int = 0, hr_resize_x: int = 0, hr_resize_y: int = 0, **kwargs):
+    def __init__(self, enable_hr: bool = False, denoising_strength: float = 0.75, firstphase_width: int = 0, firstphase_height: int = 0, hr_scale: float = 2.0, hr_upscaler: str = None, hr_second_pass_steps: int = 0, hr_resize_x: int = 0, hr_resize_y: int = 0, refiner_steps: int = 0, refiner_denoise: int = 0, refiner_prompt: str = '', refiner_negative: str = '', **kwargs):
+
         super().__init__(**kwargs)
         self.enable_hr = enable_hr
         self.denoising_strength = denoising_strength
@@ -871,6 +902,10 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
         self.truncate_x = 0
         self.truncate_y = 0
         self.applied_old_hires_behavior_to = None
+        self.refiner_steps = refiner_steps
+        self.refiner_denoise = refiner_denoise
+        self.refiner_prompt = refiner_prompt
+        self.refiner_negative = refiner_negative
 
     def init(self, all_prompts, all_seeds, all_subseeds):
         if shared.backend == Backend.DIFFUSERS:

modules/sd_models.py

@@ -538,15 +538,18 @@ class PriorPipeline:
         self.prior.to(*args, **kwargs)
 
     def enable_model_cpu_offload(self, *args, **kwargs):
-        self.prior.enable_model_cpu_offload(*args, **kwargs)
+        if hasattr(self.prior, 'enable_model_cpu_offload'):
+            self.prior.enable_model_cpu_offload(*args, **kwargs)
         self.main.enable_model_cpu_offload(*args, **kwargs)
 
     def enable_sequential_cpu_offload(self, *args, **kwargs):
-        self.prior.enable_sequential_cpu_offload(*args, **kwargs)
+        if hasattr(self.prior, 'enable_sequential_cpu_offload'):
+            self.prior.enable_sequential_cpu_offload(*args, **kwargs)
         self.main.enable_sequential_cpu_offload(*args, **kwargs)
 
     def enable_xformers_memory_efficient_attention(self, *args, **kwargs):
-        self.prior.enable_xformers_memory_efficient_attention(*args, **kwargs)
+        if hasattr(self.prior, 'enable_xformers_memory_efficient_attention'):
+            self.prior.enable_xformers_memory_efficient_attention(*args, **kwargs)
         self.main.enable_xformers_memory_efficient_attention(*args, **kwargs)
 
     def __call__(self, *args, **kwargs):
@@ -682,7 +685,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
                         shared.log.error(f'Diffusers cannot load safetensor model: {checkpoint_info.path} {shared.opts.diffusers_pipeline}')
                         return
                     if sd_model is not None:
-                        shared.log.debug(f'Diffusers pipeline: {type(sd_model)}') # pylint: disable=protected-access
+                        shared.log.debug(f'Diffusers pipeline: {sd_model.__class__.__name__}') # pylint: disable=protected-access
                 except Exception as e:
                     shared.log.error(f'Diffusers failed loading model using pipeline: {checkpoint_info.path} {shared.opts.diffusers_pipeline} {e}')
                     return
@@ -778,7 +781,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
         sd_model.sd_checkpoint_info = checkpoint_info # pylint: disable=attribute-defined-outside-init
         sd_model.sd_model_checkpoint = checkpoint_info.filename # pylint: disable=attribute-defined-outside-init
         sd_model.sd_model_hash = checkpoint_info.hash # pylint: disable=attribute-defined-outside-init
-        sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {elapsed} {remaining}', ncols=80, colour='#327fba')
+        if hasattr(sd_model, "set_progress_bar_config"):
+            sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {elapsed} {remaining}', ncols=80, colour='#327fba')
         if op == 'refiner' and shared.opts.diffusers_move_refiner:
             shared.log.debug('Moving refiner model to CPU')
             sd_model.to("cpu")
@@ -975,8 +979,8 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model')
     if sd_model is None:  # previous model load failed
         current_checkpoint_info = None
     else:
-        current_checkpoint_info = sd_model.sd_checkpoint_info
-        if checkpoint_info is not None and current_checkpoint_info.filename == checkpoint_info.filename:
+        current_checkpoint_info = getattr(sd_model, 'sd_checkpoint_info', None)
+        if current_checkpoint_info is not None and checkpoint_info is not None and current_checkpoint_info.filename == checkpoint_info.filename:
             return
         if shared.cmd_opts.lowvram or shared.cmd_opts.medvram:
             lowvram.send_everything_to_cpu()

modules/txt2img.py

@@ -5,7 +5,7 @@ from modules.ui import plaintext_to_html
 from modules.memstats import memory_stats
 
 
-def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, seed_enable_extras: bool, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, override_settings_texts, *args): # pylint: disable=unused-argument
+def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, seed_enable_extras: bool, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_steps: int, refiner_denoise: float, refiner_prompt: str, refiner_negative: str, override_settings_texts, *args): # pylint: disable=unused-argument
 
     shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}|seed_enable_extras={seed_enable_extras}|height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|override_settings_texts={override_settings_texts}args={args}')
     if sampler_index is None:
@@ -47,6 +47,10 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step
         hr_second_pass_steps=hr_second_pass_steps,
         hr_resize_x=hr_resize_x,
         hr_resize_y=hr_resize_y,
+        refiner_steps=refiner_steps,
+        refiner_denoise=refiner_denoise,
+        refiner_prompt=refiner_prompt,
+        refiner_negative=refiner_negative,
         override_settings=override_settings,
     )
     p.scripts = modules.scripts.scripts_txt2img

modules/ui.py

@@ -375,6 +375,7 @@ def create_ui(startup_timer = None):
                             restore_faces = gr.Checkbox(label='Restore faces', value=False, visible=len(modules.shared.face_restorers) > 1, elem_id="txt2img_restore_faces")
                             tiling = gr.Checkbox(label='Tiling', value=False, elem_id="txt2img_tiling")
                             enable_hr = gr.Checkbox(label='Hires fix', value=False, elem_id="txt2img_enable_hr")
+                            enable_refiner = gr.Checkbox(label='Refiner', value=False, elem_id="txt2img_enable_refiner")
                             hr_final_resolution = FormHTML(value="", elem_id="txtimg_hr_finalres", label="Upscaled resolution", interactive=False)
                     elif category == "hires_fix":
                         with FormGroup(visible=False, elem_id="txt2img_hires_fix") as hr_options:
@@ -387,6 +388,14 @@ def create_ui(startup_timer = None):
                             with FormRow(elem_id="txt2img_hires_fix_row3", variant="compact"):
                                 hr_resize_x = gr.Slider(minimum=0, maximum=2048, step=8, label="Resize width to", value=0, elem_id="txt2img_hr_resize_x")
                                 hr_resize_y = gr.Slider(minimum=0, maximum=2048, step=8, label="Resize height to", value=0, elem_id="txt2img_hr_resize_y")
+                        with FormGroup(visible=False, elem_id="txt2img_hires_fix") as refiner_options:
+                            with FormRow(elem_id="txt2img_refiner_row1", variant="compact"):
+                                refiner_steps = gr.Slider(minimum=1, maximum=99, step=1, label='Steps', value=5, elem_id="txt2img_refiner_steps")
+                                refiner_denoise = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.5, elem_id="txt2img_refiner_denoise")
+                            with FormRow(elem_id="txt2img_refiner_row2", variant="compact"):
+                                refiner_prompt = gr.Textbox(value='', label='Prompt')
+                            with FormRow(elem_id="txt2img_refiner_row3", variant="compact"):
+                                refiner_negative = gr.Textbox(value='', label='Negative prompt')
                     elif category == "override_settings":
                         with FormRow(elem_id="txt2img_override_settings_row") as row:
                             override_settings = create_override_settings_dropdown('txt2img', row)
@@ -435,6 +444,10 @@ def create_ui(startup_timer = None):
                     hr_second_pass_steps,
                     hr_resize_x,
                     hr_resize_y,
+                    refiner_steps,
+                    refiner_denoise,
+                    refiner_prompt,
+                    refiner_negative,
                     override_settings,
                 ] + custom_inputs,
                 outputs=[
@@ -451,23 +464,10 @@ def create_ui(startup_timer = None):
 
             res_switch_btn.click(lambda w, h: (h, w), inputs=[width, height], outputs=[width, height], show_progress=False)
 
-            txt_prompt_img.change(
-                fn=modules.images.image_data,
-                inputs=[
-                    txt_prompt_img
-                ],
-                outputs=[
-                    txt2img_prompt,
-                    txt_prompt_img
-                ]
-            )
+            txt_prompt_img.change(fn=modules.images.image_data, inputs=[txt_prompt_img], outputs=[txt2img_prompt, txt_prompt_img])
 
-            enable_hr.change(
-                fn=lambda x: gr_show(x),
-                inputs=[enable_hr],
-                outputs=[hr_options],
-                show_progress = False,
-            )
+            enable_hr.change(fn=lambda x: gr_show(x), inputs=[enable_hr], outputs=[hr_options], show_progress = False)
+            enable_refiner.change(fn=lambda x: gr_show(x), inputs=[enable_refiner], outputs=[refiner_options], show_progress = False)
 
             txt2img_paste_fields = [
                 (txt2img_prompt, "Prompt"),

requirements.txt

@@ -49,7 +49,7 @@ requests==2.31.0
 tqdm==4.65.0
 accelerate==0.20.3
 opencv-python==4.7.0.72
-diffusers==0.18.1
+diffusers==0.18.2
 einops==0.4.1
 gradio==3.32.0
 numexpr==2.8.4