From 5c8ead7be041ed1aa59afcf66694cd7b0caaedb7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 12 Jul 2023 15:35:36 -0400 Subject: [PATCH] update diffusers --- CHANGELOG.md | 18 ++++++++- TODO.md | 43 +++++++++++---------- modules/processing.py | 89 ++++++++++++++++++++++++++++++------------- modules/sd_models.py | 18 +++++---- modules/txt2img.py | 6 ++- modules/ui.py | 32 ++++++++-------- requirements.txt | 2 +- 7 files changed, 134 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e98307f2c..7a9e5ea70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,22 @@ # Change Log for SD.Next -## Update for 07/11/2023: +## Update for 07/12/2023 + +Another big one... + +- diffusers backend: + - separate ui settings for refiner pass with sd-xl + you can specify: prompt, negative prompt, steps, denoise start + - fix loading from pure safetensors files + now you can load sd-xl from safetensors file or from huggingface folder format + - fix kandinsky model +- other: + - major refactoring of the javascript code + includes fixes for text selections and navigation + - minor fixes in extra-networks + +big thanks to @huggingface team for great communication, support and fixing all the reported issues asap! -- fixes in extra-networks, diffusers samplers ## Update for 07/10/2023 diff --git a/TODO.md b/TODO.md index 86dd63e14..ce756c19b 100644 --- a/TODO.md +++ b/TODO.md @@ -2,12 +2,19 @@ ## Issues -Stuff to be fixed... +Stuff to be fixed, in no particular order... +- SD-XL Lora +- SD-XL Img2Img/Inpaint +- Kandinsky 2.2 (2.1 is working) +- Refresh `sd_checkpoint` pulldown on backend switch +- Extensions misteriously auto-enabling +- VAE/UNet dtypes +- Attention head ## Features -Stuff to be added... +Stuff to be added, in no particular order... - Update `Wiki` - Create new `GitHub` hooks/actions for CI/CD @@ -15,8 +22,19 @@ Stuff to be added... - Update `train.py` to use `interrogator` - Update `train.py` to use `rembg` - Create new train UI -- Create new Models UI -- Intelligent preview mode +- Docker PR +- Port `p.all_hr_prompts` +- Image watermark using `image-watermark` +- Image phash and hdash using `imagehash` +- Model merge using `git-rebasin` +- Additional upscalers +- New image browser +- Update `gradio` +- Rename repo: **automatic** -> **sdnext** +- New icons +- Enable refiner workflow for `ldm` backend +- Improve `lyco` logging +- Cache models when switching backends ## Investigate @@ -49,19 +67,4 @@ Tech that can be integrated as part of the core workflow... - Bunch of stuff: - -- docker -- port `p.all_hr_prompts` -- test `lyco_patch_lora` -- fix `lyco` logging -- image watermark -- image `imagehash` phash and hdash -- git-rebasin -- additional upscalers -- new image browser -- `git submodule set-url extensions-builtin/clip-interrogator-ext https://github.com/Dahvikiin/clip-interrogator-ext.git` -- upate `gradio` -- extra network refresh breaks if new extra network type found -- [sd-xl lora](https://civitai.com/models/104913/fcstyledxl) -- sd-xl img2img with configurable steps -- change backend pipeline on-the-fly -- rename repo + diff --git a/modules/processing.py b/modules/processing.py index 378b569bf..5ef1dd226 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -3,6 +3,7 @@ import math import os import hashlib import random +import inspect from contextlib import nullcontext from typing import Any, Dict, List import torch @@ -612,6 +613,45 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: cache[0] = (required_prompts, steps) return cache[1] + # TODO Diffusers limited callbacks + def diffusers_callback(step: int, _timestep: int, latents: torch.FloatTensor): + shared.state.sampling_step = step + shared.state.sampling_steps = p.steps + shared.state.current_latent = latents + shared.state.set_current_image() + + def set_pipeline_args(model, prompt, negative_prompt, **kwargs): + args = {} + pipeline = model.main if model.__class__.__name__ == 'PriorPipeline' else model + signature = inspect.signature(type(pipeline).__call__) + possible = signature.parameters.keys() + generator_device = 'cpu' if shared.opts.diffusers_generator_device == "cpu" else shared.device + generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds] + if 'prompt' in possible: + args['prompt'] = prompt + if 'negative_prompt' in possible: + args['negative_prompt'] = negative_prompt + if 'num_inference_steps' in possible: + args['num_inference_steps'] = p.steps + if 'guidance_scale' in possible: + args['guidance_scale'] = p.cfg_scale + if 'generator' in possible: + args['generator'] = generator + if 'output_type' in possible: + args['output_type'] = 'np' + if 'callback_steps' in possible: + args['callback_steps'] = 1 + if 'callback' in args: + args['callback'] = diffusers_callback + if 'cross_attention_kwargs' in possible: + args['cross_attention_kwargs'] = cross_attention_kwargs + for arg in kwargs: + if arg in possible: + args[arg] = kwargs[arg] + log.debug(f'Diffuser pipeline: {pipeline.__class__.__name__} args={args.keys()}') + return args + + ema_scope_context = p.sd_model.ema_scope if shared.backend == Backend.ORIGINAL else nullcontext with torch.no_grad(), ema_scope_context(): with devices.autocast(): @@ -682,8 +722,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: del samples_ddim elif shared.backend == Backend.DIFFUSERS: - generator_device = 'cpu' if shared.opts.diffusers_generator_device == "cpu" else shared.device - generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds] if (not hasattr(shared.sd_model.scheduler, 'name')) or (shared.sd_model.scheduler.name != p.sampler_name): sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) if sampler is None: @@ -702,41 +740,33 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: # TODO(PVP): change out to latents once possible with `diffusers` task_specific_kwargs = {"image": p.init_images[0], "mask_image": p.image_mask, "strength": p.denoising_strength} - # TODO Diffusers limited callbacks # TODO Diffusers processing is not using p.sample so second pass is ignored - def diffusers_callback(step: int, _timestep: int, latents: torch.FloatTensor): - shared.state.sampling_step = step - shared.state.sampling_steps = p.steps - shared.state.current_latent = latents - shared.state.set_current_image() shared.sd_model.to(devices.device) - - pipe_args = { # TODO needs dynamic discovery of possible args - "prompt": prompts, - "negative_prompt": negative_prompts, - "num_inference_steps": p.steps, - "guidance_scale": p.cfg_scale, - "generator": generator, - "output_type": 'np' if shared.sd_refiner is None else 'latent', - "callback_steps": 1, # TODO not supported by Kandinsky - "callback": diffusers_callback, # TODO not supported by Kandinsky - "cross_attention_kwargs": cross_attention_kwargs, # TODO not supported by Kandinsky - } - output = shared.sd_model(**pipe_args, **task_specific_kwargs) # pylint: disable=not-callable - + pipe_args = set_pipeline_args( + model=shared.sd_model, + prompt=prompts, + negative_prompt=negative_prompts, + output_type='np' if shared.sd_refiner is None else 'latent', + **task_specific_kwargs + ) + output = shared.sd_model(**pipe_args) # pylint: disable=not-callable if shared.sd_refiner is not None: if shared.opts.diffusers_move_base: shared.log.debug('Moving base model to CPU') shared.sd_model.to('cpu') shared.sd_refiner.to(devices.device) devices.torch_gc() - init_image = output.images[0] - pipe_args['image'] = init_image - pipe_args['output_type'] = 'np' + pipe_args = set_pipeline_args( + model=shared.sd_refiner, + prompt=p.refiner_prompt if len(p.refiner_prompt) > 0 else prompts, + negative_prompt=p.refiner_negative if len(p.refiner_negative) > 0 else negative_prompts, + image=output.images[0], + output_type='np' + ) output = shared.sd_refiner(**pipe_args) # pylint: disable=not-callable if shared.opts.diffusers_move_refiner: - shared.log.debug('Moving refiner model to CPU') + log.debug('Moving refiner model to CPU') shared.sd_refiner.to('cpu') x_samples_ddim = output.images @@ -852,7 +882,8 @@ def old_hires_fix_first_pass_dimensions(width, height): class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): sampler = None - def __init__(self, enable_hr: bool = False, denoising_strength: float = 0.75, firstphase_width: int = 0, firstphase_height: int = 0, hr_scale: float = 2.0, hr_upscaler: str = None, hr_second_pass_steps: int = 0, hr_resize_x: int = 0, hr_resize_y: int = 0, **kwargs): + def __init__(self, enable_hr: bool = False, denoising_strength: float = 0.75, firstphase_width: int = 0, firstphase_height: int = 0, hr_scale: float = 2.0, hr_upscaler: str = None, hr_second_pass_steps: int = 0, hr_resize_x: int = 0, hr_resize_y: int = 0, refiner_steps: int = 0, refiner_denoise: int = 0, refiner_prompt: str = '', refiner_negative: str = '', **kwargs): + super().__init__(**kwargs) self.enable_hr = enable_hr self.denoising_strength = denoising_strength @@ -871,6 +902,10 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.truncate_x = 0 self.truncate_y = 0 self.applied_old_hires_behavior_to = None + self.refiner_steps = refiner_steps + self.refiner_denoise = refiner_denoise + self.refiner_prompt = refiner_prompt + self.refiner_negative = refiner_negative def init(self, all_prompts, all_seeds, all_subseeds): if shared.backend == Backend.DIFFUSERS: diff --git a/modules/sd_models.py b/modules/sd_models.py index f322e58ab..0cc522410 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -538,15 +538,18 @@ class PriorPipeline: self.prior.to(*args, **kwargs) def enable_model_cpu_offload(self, *args, **kwargs): - self.prior.enable_model_cpu_offload(*args, **kwargs) + if hasattr(self.prior, 'enable_model_cpu_offload'): + self.prior.enable_model_cpu_offload(*args, **kwargs) self.main.enable_model_cpu_offload(*args, **kwargs) def enable_sequential_cpu_offload(self, *args, **kwargs): - self.prior.enable_sequential_cpu_offload(*args, **kwargs) + if hasattr(self.prior, 'enable_sequential_cpu_offload'): + self.prior.enable_sequential_cpu_offload(*args, **kwargs) self.main.enable_sequential_cpu_offload(*args, **kwargs) def enable_xformers_memory_efficient_attention(self, *args, **kwargs): - self.prior.enable_xformers_memory_efficient_attention(*args, **kwargs) + if hasattr(self.prior, 'enable_xformers_memory_efficient_attention'): + self.prior.enable_xformers_memory_efficient_attention(*args, **kwargs) self.main.enable_xformers_memory_efficient_attention(*args, **kwargs) def __call__(self, *args, **kwargs): @@ -682,7 +685,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No shared.log.error(f'Diffusers cannot load safetensor model: {checkpoint_info.path} {shared.opts.diffusers_pipeline}') return if sd_model is not None: - shared.log.debug(f'Diffusers pipeline: {type(sd_model)}') # pylint: disable=protected-access + shared.log.debug(f'Diffusers pipeline: {sd_model.__class__.__name__}') # pylint: disable=protected-access except Exception as e: shared.log.error(f'Diffusers failed loading model using pipeline: {checkpoint_info.path} {shared.opts.diffusers_pipeline} {e}') return @@ -778,7 +781,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.sd_checkpoint_info = checkpoint_info # pylint: disable=attribute-defined-outside-init sd_model.sd_model_checkpoint = checkpoint_info.filename # pylint: disable=attribute-defined-outside-init sd_model.sd_model_hash = checkpoint_info.hash # pylint: disable=attribute-defined-outside-init - sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {elapsed} {remaining}', ncols=80, colour='#327fba') + if hasattr(sd_model, "set_progress_bar_config"): + sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {elapsed} {remaining}', ncols=80, colour='#327fba') if op == 'refiner' and shared.opts.diffusers_move_refiner: shared.log.debug('Moving refiner model to CPU') sd_model.to("cpu") @@ -975,8 +979,8 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model') if sd_model is None: # previous model load failed current_checkpoint_info = None else: - current_checkpoint_info = sd_model.sd_checkpoint_info - if checkpoint_info is not None and current_checkpoint_info.filename == checkpoint_info.filename: + current_checkpoint_info = getattr(sd_model, 'sd_checkpoint_info', None) + if current_checkpoint_info is not None and checkpoint_info is not None and current_checkpoint_info.filename == checkpoint_info.filename: return if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: lowvram.send_everything_to_cpu() diff --git a/modules/txt2img.py b/modules/txt2img.py index 8ca3af5f9..68c470213 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -5,7 +5,7 @@ from modules.ui import plaintext_to_html from modules.memstats import memory_stats -def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, seed_enable_extras: bool, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, override_settings_texts, *args): # pylint: disable=unused-argument +def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, seed_enable_extras: bool, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_steps: int, refiner_denoise: float, refiner_prompt: str, refiner_negative: str, override_settings_texts, *args): # pylint: disable=unused-argument shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}|seed_enable_extras={seed_enable_extras}|height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|override_settings_texts={override_settings_texts}args={args}') if sampler_index is None: @@ -47,6 +47,10 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step hr_second_pass_steps=hr_second_pass_steps, hr_resize_x=hr_resize_x, hr_resize_y=hr_resize_y, + refiner_steps=refiner_steps, + refiner_denoise=refiner_denoise, + refiner_prompt=refiner_prompt, + refiner_negative=refiner_negative, override_settings=override_settings, ) p.scripts = modules.scripts.scripts_txt2img diff --git a/modules/ui.py b/modules/ui.py index de990a574..6134b1a07 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -375,6 +375,7 @@ def create_ui(startup_timer = None): restore_faces = gr.Checkbox(label='Restore faces', value=False, visible=len(modules.shared.face_restorers) > 1, elem_id="txt2img_restore_faces") tiling = gr.Checkbox(label='Tiling', value=False, elem_id="txt2img_tiling") enable_hr = gr.Checkbox(label='Hires fix', value=False, elem_id="txt2img_enable_hr") + enable_refiner = gr.Checkbox(label='Refiner', value=False, elem_id="txt2img_enable_refiner") hr_final_resolution = FormHTML(value="", elem_id="txtimg_hr_finalres", label="Upscaled resolution", interactive=False) elif category == "hires_fix": with FormGroup(visible=False, elem_id="txt2img_hires_fix") as hr_options: @@ -387,6 +388,14 @@ def create_ui(startup_timer = None): with FormRow(elem_id="txt2img_hires_fix_row3", variant="compact"): hr_resize_x = gr.Slider(minimum=0, maximum=2048, step=8, label="Resize width to", value=0, elem_id="txt2img_hr_resize_x") hr_resize_y = gr.Slider(minimum=0, maximum=2048, step=8, label="Resize height to", value=0, elem_id="txt2img_hr_resize_y") + with FormGroup(visible=False, elem_id="txt2img_hires_fix") as refiner_options: + with FormRow(elem_id="txt2img_refiner_row1", variant="compact"): + refiner_steps = gr.Slider(minimum=1, maximum=99, step=1, label='Steps', value=5, elem_id="txt2img_refiner_steps") + refiner_denoise = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.5, elem_id="txt2img_refiner_denoise") + with FormRow(elem_id="txt2img_refiner_row2", variant="compact"): + refiner_prompt = gr.Textbox(value='', label='Prompt') + with FormRow(elem_id="txt2img_refiner_row3", variant="compact"): + refiner_negative = gr.Textbox(value='', label='Negative prompt') elif category == "override_settings": with FormRow(elem_id="txt2img_override_settings_row") as row: override_settings = create_override_settings_dropdown('txt2img', row) @@ -435,6 +444,10 @@ def create_ui(startup_timer = None): hr_second_pass_steps, hr_resize_x, hr_resize_y, + refiner_steps, + refiner_denoise, + refiner_prompt, + refiner_negative, override_settings, ] + custom_inputs, outputs=[ @@ -451,23 +464,10 @@ def create_ui(startup_timer = None): res_switch_btn.click(lambda w, h: (h, w), inputs=[width, height], outputs=[width, height], show_progress=False) - txt_prompt_img.change( - fn=modules.images.image_data, - inputs=[ - txt_prompt_img - ], - outputs=[ - txt2img_prompt, - txt_prompt_img - ] - ) + txt_prompt_img.change(fn=modules.images.image_data, inputs=[txt_prompt_img], outputs=[txt2img_prompt, txt_prompt_img]) - enable_hr.change( - fn=lambda x: gr_show(x), - inputs=[enable_hr], - outputs=[hr_options], - show_progress = False, - ) + enable_hr.change(fn=lambda x: gr_show(x), inputs=[enable_hr], outputs=[hr_options], show_progress = False) + enable_refiner.change(fn=lambda x: gr_show(x), inputs=[enable_refiner], outputs=[refiner_options], show_progress = False) txt2img_paste_fields = [ (txt2img_prompt, "Prompt"), diff --git a/requirements.txt b/requirements.txt index 59cc27e6a..7722d9577 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,7 +49,7 @@ requests==2.31.0 tqdm==4.65.0 accelerate==0.20.3 opencv-python==4.7.0.72 -diffusers==0.18.1 +diffusers==0.18.2 einops==0.4.1 gradio==3.32.0 numexpr==2.8.4