From a2caafe4df486afc51ae141c559ac3539ba56d4a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 2 Jul 2023 14:04:54 -0400 Subject: [PATCH 01/42] initial diffusers merge into dev --- DIFFUSERS.md | 49 +++++++++ modules/modelloader.py | 53 ++++++--- modules/processing.py | 58 ++++++++-- modules/sd_hijack.py | 6 +- modules/sd_models.py | 179 +++++++++++++++++++++++++++---- modules/sd_samplers.py | 4 +- modules/sd_samplers_diffusers.py | 37 +++++++ modules/shared.py | 82 +++++++++++++- modules/ui.py | 2 +- modules/ui_models.py | 12 +++ requirements.txt | 1 + 11 files changed, 431 insertions(+), 52 deletions(-) create mode 100644 DIFFUSERS.md create mode 100644 modules/sd_samplers_diffusers.py diff --git a/DIFFUSERS.md b/DIFFUSERS.md new file mode 100644 index 000000000..199623c4d --- /dev/null +++ b/DIFFUSERS.md @@ -0,0 +1,49 @@ +# Diffusers WiP + +initial support merged into `dev` branch + + git clone https://github.com/vladmandic/automatic -b dev diffusers + cd diffusers + webui --debug --backend diffusers + +default sd 1.5 model will be downloaded automatically to `models/Diffusers` + +on first startup, disable **controlnet** and **multi-diffusion** extensions as right now they are not compatible with diffusers + +to update repo, do not use `--upgrade` flag, use manual `git pull` instead + +## Test + +### Standard + +- run with `webui --debug --backend original` +- goal is to test standard workflows (so not diffusers) to ensure there are no regressions + so diffusers code can be merged into `master` and we can continue with development there + +### Diffusers + +- sd 1.5 and sd 2.1 model + models can be downloaded from huggingface hub + but focus on default model for now and i'll add downloader soon +- lora, textual inversion + only loras/textual-inversions downloaded from huggingface hub are supported for now + i'll add standard safetensors soon +- txt2img, img2img, inpaint, outpaint, process + +### Experimental + +- cuda model compile using `reduce overhead` model with and without `fullgraph` +- kandinsky model + +## Todo + +- enable loading of safetensors models +- cleanup logging +- search&download models from hfhub +- controlnet extension +- multidiffusion extension +- sdxl model + +## Issues + +- TBD diff --git a/modules/modelloader.py b/modules/modelloader.py index c49cdeb5d..799537240 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -1,6 +1,8 @@ import os import shutil import importlib +import json +from typing import Dict from urllib.parse import urlparse from modules import shared @@ -9,29 +11,54 @@ from modules.paths import script_path, models_path diffuser_repos = [] -def load_diffusers(model_path: str, hub_url: str = None, command_path: str = None): - import huggingface_hub as hf +def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config: Dict[str, str] = None): from diffusers import DiffusionPipeline + import huggingface_hub as hf + if download_config is None: + download_config = { + "force_download": False, + "resume_download": True, + "cache_dir": shared.opts.diffusers_dir, + } + + if cache_dir is not None: + download_config["cache_dir"] = cache_dir + + pipeline_dir = DiffusionPipeline.download(hub_id, **download_config) + model_info_dict = hf.model_info(hub_id).cardData # TODO hfhub card-data? + + # some checkpoints need to be downloaded as "hidden" as they just serve as pre- or post-pipelines of other pipelines + if model_info_dict is not None and "prior" in model_info_dict: + download_dir = DiffusionPipeline.download(model_info_dict["prior"], **download_config) + model_info_dict["prior"] = download_dir + # mark prior as hidden + with open(os.path.join(download_dir, "hidden"), "w", encoding="utf-8") as f: + f.write("True") + + with open(os.path.join(pipeline_dir, "model_info.json"), "w", encoding="utf-8") as json_file: + json.dump(model_info_dict, json_file) + + return pipeline_dir + +def load_diffusers_models(model_path: str, command_path: str = None): + import huggingface_hub as hf places = [] - - # download repo - if hub_url is not None: - DiffusionPipeline.download(hub_url, cache_dir=model_path) - places.append(model_path) if command_path is not None and command_path != model_path and os.path.isdir(command_path): places.append(command_path) diffuser_repos.clear() output = [] - try: - for place in places: + for place in places: + try: res = hf.scan_cache_dir(cache_dir=place) for r in list(res.repos): - diffuser_repos.append({ 'name': r.repo_id, 'filename': r.repo_id, 'path': str(r.repo_path), 'size': r.size_on_disk, 'mtime': r.last_modified, 'hash': list(r.revisions)[-1].commit_hash }) - output.append(str(r.repo_id)) - except Exception as e: - shared.log.error(f"Error listing diffusers: {place} {e}") + cache_path = os.path.join(r.repo_path, "snapshots", list(r.revisions)[-1].commit_hash) + diffuser_repos.append({ 'name': r.repo_id, 'filename': r.repo_id, 'path': cache_path, 'size': r.size_on_disk, 'mtime': r.last_modified, 'hash': list(r.revisions)[-1].commit_hash, 'model_info': str(os.path.join(cache_path, "model_info.json")) }) + if not os.path.isfile(os.path.join(cache_path, "hidden")): + output.append(str(r.repo_id)) + except Exception as e: + shared.log.error(f"Error listing diffusers: {place} {e}") shared.log.debug(f'Scanning diffusers cache: {len(output)} {model_path} {command_path}') return output diff --git a/modules/processing.py b/modules/processing.py index 6d05beed4..e8808a910 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -223,7 +223,7 @@ class StableDiffusionProcessing: # HACK: Using introspection as the Depth2Image model doesn't appear to uniquely # identify itself with a field common to all models. The conditioning_key is also hybrid. if backend == Backend.DIFFUSERS: # TODO: Diffusers img2img_image_conditioning - return latent_image.new_zeros(latent_image.shape[0], 5, 1, 1) + return None if isinstance(self.sd_model, LatentDepth2ImageDiffusion): return self.depth2img_image_conditioning(source_image) if self.sd_model.cond_stage_key == "edit": @@ -520,7 +520,8 @@ def process_images(p: StableDiffusionProcessing) -> Processed: if k == 'sd_vae': sd_vae.reload_vae_weights() - sd_models.apply_token_merging(p.sd_model, p.get_token_merging_ratio()) + if not shared.opts.cuda_compile: + sd_models.apply_token_merging(p.sd_model, p.get_token_merging_ratio()) if cmd_opts.profile: """ @@ -538,7 +539,8 @@ def process_images(p: StableDiffusionProcessing) -> Processed: else: res = process_images_inner(p) finally: - sd_models.apply_token_merging(p.sd_model, 0) + if not shared.opts.cuda_compile: + sd_models.apply_token_merging(p.sd_model, 0) if p.override_settings_restore_afterwards: # restore opts to original state for k, v in stored_opts.items(): setattr(opts, k, v) @@ -557,6 +559,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: assert len(p.prompt) > 0 else: assert p.prompt is not None + seed = get_fixed_seed(p.seed) subseed = get_fixed_seed(p.subseed) if backend == Backend.ORIGINAL: @@ -683,26 +686,39 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: devices.torch_gc() if p.scripts is not None: p.scripts.postprocess_batch(p, x_samples_ddim, batch_number=n) - else: # TODO Diffusers main processing + + elif backend == Backend.DIFFUSERS: generator = [torch.Generator(device="cpu").manual_seed(s) for s in seeds] if shared.sd_model.scheduler.name != p.sampler_name: sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") scheduler = sampler.constructor(shared.sd_model.sd_checkpoint_info.filename) + # TODO(Patrick): For wrapped pipelines this is currently a no-op shared.sd_model.scheduler = scheduler.sampler + + if sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE: + task_specific_kwargs = {"height": p.height, "width": p.width} + elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE: + task_specific_kwargs = {"image": p.init_images[0], "strength": p.denoising_strength} + elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING: + # TODO(PVP): change out to latents once possible with `diffusers` + task_specific_kwargs = {"image": p.init_images[0], "mask_image": p.image_mask, "strength": p.denoising_strength} + output = shared.sd_model( prompt=prompts, negative_prompt=negative_prompts, num_inference_steps=p.steps, guidance_scale=p.cfg_scale, - height=p.height, - width=p.width, generator=generator, output_type="np", + **task_specific_kwargs ) x_samples_ddim = output.images + else: + raise ValueError(f"Unknown backend {backend}") + for i, x_sample in enumerate(x_samples_ddim): p.batch_index = i if backend == Backend.ORIGINAL: @@ -820,8 +836,12 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.applied_old_hires_behavior_to = None def init(self, all_prompts, all_seeds, all_subseeds): + if backend == Backend.DIFFUSERS: + sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) + self.width = self.width or 512 self.height = self.height or 512 + if self.enable_hr: if opts.use_old_hires_fix_width_height and self.applied_old_hires_behavior_to != (self.width, self.height): self.hr_resize_x = self.width @@ -873,6 +893,9 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.extra_generation_params["Hires upscaler"] = self.hr_upscaler def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): # TODO this is majority of processing time + if backend == Backend.DIFFUSERS: + sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) + self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model) latent_scale_mode = shared.latent_upscale_modes.get(self.hr_upscaler, None) if self.hr_upscaler is not None else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "nearest") if self.enable_hr and latent_scale_mode is None: @@ -978,12 +1001,18 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): self.image_conditioning = None def init(self, all_prompts, all_seeds, all_subseeds): + image_mask = self.image_mask + if backend == Backend.DIFFUSERS and image_mask is None: + sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) + elif backend == Backend.DIFFUSERS and image_mask is not None: + sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.INPAINTING) + self.sd_model.dtype = self.sd_model.unet.dtype + force_latent_upscaler = shared.opts.data.get('force_latent_sampler') if self.sampler_name in ['PLMS']: self.sampler_name = force_latent_upscaler if force_latent_upscaler != 'None' else shared.opts.fallback_sampler # PLMS does not support img2img, use fallback instead self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model) crop_region = None - image_mask = self.image_mask if image_mask is not None: image_mask = image_mask.convert('L') if self.inpainting_mask_invert: @@ -1048,7 +1077,13 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): image = torch.from_numpy(batch_images) image = 2. * image - 1. image = image.to(shared.device) - self.init_latent = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(image)) + + if backend == Backend.ORIGINAL: + self.init_latent = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(image)) + else: + # we don't pre-encode the latents for diffusers to allow the UI to stay general for different model types + self.init_latent = None + if self.resize_mode == 3: self.init_latent = torch.nn.functional.interpolate(self.init_latent, size=(self.height // opt_f, self.width // opt_f), mode="bilinear") if image_mask is not None: @@ -1068,6 +1103,13 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): self.image_conditioning = self.img2img_image_conditioning(image, self.init_latent, image_mask) def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): + if backend == Backend.DIFFUSERS: + if self.init_mask is None: # pylint: disable=no-member + sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) + else: + sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.INPAINTING) + self.sd_model.dtype = self.sd_model.unet.dtype + x = create_random_tensors([opt_C, self.height // opt_f, self.width // opt_f], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self) if self.initial_noise_multiplier != 1.0: self.extra_generation_params["Noise multiplier"] = self.initial_noise_multiplier diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 420fbcf18..5e397ebb9 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -179,11 +179,11 @@ class StableDiffusionModelHijack: shared.log.info("Model compile enabled: IPEX Optimize Graph Mode") else: shared.log.warning("Model compile skipped: IPEX Method is for Intel GPU's with OneAPI") - elif opts.cuda_compile and opts.cuda_compile_mode != 'none': + elif opts.cuda_compile and opts.cuda_compile_mode != 'none' and shared.backend == shared.Backend.ORIGINAL: try: import logging import torch._dynamo as dynamo # pylint: disable=unused-import - torch._dynamo.config.log_level = logging.WARNING if opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access + # torch._dynamo.config.log_level = logging.WARNING if opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access torch._dynamo.config.verbose = opts.cuda_compile_verbose # pylint: disable=protected-access torch._dynamo.config.suppress_errors = opts.cuda_compile_errors # pylint: disable=protected-access torch.backends.cudnn.benchmark = True @@ -191,7 +191,7 @@ class StableDiffusionModelHijack: import hidet hidet.torch.dynamo_config.use_tensor_core(True) hidet.torch.dynamo_config.search_space(2) - m.model = torch.compile(m.model, mode="default", backend=opts.cuda_compile_mode, fullgraph=False, dynamic=False) + m.model = torch.compile(m.model, mode="default", backend=opts.cuda_compile_mode, fullgraph=opts.cuda_compile_fullgraph, dynamic=False) shared.log.info(f"Model compile enabled: {opts.cuda_compile_mode}") except Exception as err: shared.log.warning(f"Model compile not supported: {err}") diff --git a/modules/sd_models.py b/modules/sd_models.py index 30d5dfdb8..d237f8728 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -6,6 +6,7 @@ import json import threading from os import mkdir from urllib import request +from enum import Enum import filelock from rich import progress # pylint: disable=redefined-builtin import torch @@ -13,6 +14,7 @@ import safetensors.torch from omegaconf import OmegaConf import tomesd from transformers import logging as transformers_logging +import diffusers import ldm.modules.midas as midas from ldm.util import instantiate_from_config from modules import paths, shared, modelloader, devices, script_callbacks, sd_vae, sd_disable_initialization, errors, hashes, sd_models_config @@ -52,7 +54,7 @@ class CheckpointInfo: self.name = name self.hash = model_hash(self.filename) self.sha256 = hashes.sha256_from_cache(self.filename, f"checkpoint/{name}") - else: # TODO Diffusers + elif shared.backend == shared.Backend.DIFFUSERS: repo = [r for r in modelloader.diffuser_repos if filename == r['filename']] if len(repo) == 0: error_message = f'Cannot find diffuser model: {filename}' @@ -61,6 +63,17 @@ class CheckpointInfo: self.name = repo[0]['name'] self.hash = repo[0]['hash'][:8] self.sha256 = repo[0]['hash'] + self.path = repo[0]['path'] + + if os.path.isfile(repo[0]['model_info']): + file_path = repo[0]['model_info'] + with open(file_path, "r", encoding="utf-8") as json_file: + self.model_info = json.load(json_file) + else: + self.model_info = None + else: + raise ValueError(f'Unknown backend: {shared.backend}') + self.name_for_extra = os.path.splitext(os.path.basename(filename))[0] self.model_name = os.path.splitext(name.replace("/", "_").replace("\\", "_"))[0] self.shorthash = self.sha256[0:10] if self.sha256 else None @@ -116,7 +129,8 @@ def list_models(): else: global model_path # pylint: disable=global-statement model_path = os.path.join(models_path, 'Diffusers') - model_list = modelloader.load_diffusers(model_path=model_path, command_path=shared.opts.diffusers_dir) + model_list = modelloader.load_diffusers_models(model_path=model_path, command_path=shared.opts.diffusers_dir) + for filename in sorted(model_list, key=str.lower): checkpoint_info = CheckpointInfo(filename) if checkpoint_info.name is not None: @@ -143,15 +157,15 @@ def list_models(): shared.opts.data['sd_model_checkpoint'] = "v1-5-pruned-emaonly.safetensors" model_list = modelloader.load_models(model_path=model_path, model_url=model_url, command_path=shared.opts.ckpt_dir, ext_filter=[".ckpt", ".safetensors"], download_name="v1-5-pruned-emaonly.safetensors", ext_blacklist=[".vae.ckpt", ".vae.safetensors"]) else: - hub_url = "runwayml/stable-diffusion-v1-5" - model_list = modelloader.load_diffusers(model_path=model_path, hub_url=hub_url, command_path=shared.opts.diffusers_dir) + default_model_id = "runwayml/stable-diffusion-v1-5" + modelloader.download_diffusers_model(default_model_id, model_path) + model_list = modelloader.load_diffusers_models(model_path=model_path, command_path=shared.opts.diffusers_dir) for filename in sorted(model_list, key=str.lower): checkpoint_info = CheckpointInfo(filename) if checkpoint_info.name is not None: checkpoint_info.register() - def update_model_hashes(): txt = [] lst = [ckpt for ckpt in checkpoints_list.values() if ckpt.hash is None] @@ -474,49 +488,115 @@ class SdModelData: model_data = SdModelData() +class PriorPipeline: + def __init__(self, prior, main): + self.prior = prior + self.main = main + self.scheduler = main.scheduler + self.tokenizer = self.prior.tokenizer + + def to(self, *args, **kwargs): + # only the prior is moved to CUDA in a first step + self.prior.to(*args, **kwargs) + + def enable_model_cpu_offload(self, *args, **kwargs): + self.prior.enable_model_cpu_offload(*args, **kwargs) + self.main.enable_model_cpu_offload(*args, **kwargs) + + def enable_sequential_cpu_offload(self, *args, **kwargs): + self.prior.enable_sequential_cpu_offload(*args, **kwargs) + self.main.enable_sequential_cpu_offload(*args, **kwargs) + + def enable_xformers_memory_efficient_attention(self, *args, **kwargs): + self.prior.enable_xformers_memory_efficient_attention(*args, **kwargs) + self.main.enable_xformers_memory_efficient_attention(*args, **kwargs) + + def __call__(self, *args, **kwargs): + unclip_outputs = self.prior(prompt=kwargs.get("prompt"), negative_prompt=kwargs.get("negative_prompt")) + + if self.prior.device.type == "cuda": + prior_device = self.prior.device + self.prior.to("cpu") + self.main.to(prior_device) + + kwargs = {**kwargs, **unclip_outputs} + result = self.main(*args, **kwargs) + + if self.main.device.type == "cuda": + main_device = self.main.device + self.main.to("cpu") + self.prior.to(main_device) + + return result + + def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None): # pylint: disable=unused-argument if timer is None: timer = Timer() - import diffusers import logging logging.getLogger("diffusers").setLevel(logging.ERROR) timer.record("diffusers") - diffusor_config = { - "force_download": False, - "safety_checker": None, - "resume_download": True, + diffusers_load_config = { "low_cpu_mem_usage": True, - "use_safetensors": True, - "cache_dir": shared.opts.diffusers_dir, "torch_dtype": devices.dtype, + "safety_checker": None, + # "use_safetensors": True, # TODO(PVP) - we can't enable this for all checkpoints just yet } + if shared.opts.data['sd_model_checkpoint'] == 'model.ckpt': shared.opts.data['sd_model_checkpoint'] = "runwayml/stable-diffusion-v1-5" sd_model = None try: if shared.cmd_opts.ckpt is not None and model_data.initial: # initial load model_name = modelloader.find_diffuser(shared.cmd_opts.ckpt) + if model_name is not None: shared.log.info(f'Loading diffuser model: {model_name}') - scheduler = diffusers.UniPCMultistepScheduler.from_pretrained(model_name, subfolder="scheduler") - sd_model = diffusers.DiffusionPipeline.from_pretrained(model_name, scheduler=scheduler, **diffusor_config) + model_file = modelloader.download_diffusers_model(hub_id=model_name) + sd_model = diffusers.DiffusionPipeline.from_pretrained(model_file, **diffusers_load_config) + list_models() # rescan for downloaded model checkpoint_info = CheckpointInfo(model_name) + if sd_model is None: checkpoint_info = checkpoint_info or select_checkpoint() shared.log.info(f'Loading diffuser model: {checkpoint_info.filename}') - scheduler = diffusers.UniPCMultistepScheduler.from_pretrained(checkpoint_info.filename, subfolder="scheduler") - sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.filename, scheduler=scheduler, **diffusor_config) + sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) + + if "StableDiffusion" in sd_model.__class__.__name__: + sd_model.scheduler = diffusers.UniPCMultistepScheduler.from_config(sd_model.scheduler.config) + sd_model.scheduler.name = 'UniPC' + elif "Kandinsky" in sd_model.__class__.__name__: + sd_model.scheduler.name = 'DDIM' + + # Prior pipelines + if checkpoint_info.model_info is not None and "prior" in checkpoint_info.model_info: + prior_id = checkpoint_info.model_info["prior"] + shared.log.info(f"Loading prior {prior_id} for {checkpoint_info.filename}") + prior = diffusers.DiffusionPipeline.from_pretrained(prior_id, **diffusers_load_config) + sd_model = PriorPipeline(prior=prior, main=sd_model) # wrap sd_model + if shared.cmd_opts.medvram: sd_model.enable_model_cpu_offload() if shared.cmd_opts.lowvram: sd_model.enable_sequential_cpu_offload() if shared.opts.cross_attention_optimization == "xFormers": sd_model.enable_xformers_memory_efficient_attention() - sd_model.sd_checkpoint_info = checkpoint_info - sd_model.sd_model_checkpoint = checkpoint_info.filename - sd_model.sd_model_hash = checkpoint_info.hash - scheduler.name = 'UniPC' + if shared.opts.cuda_compile and torch.cuda.is_available(): + sd_model.to(devices.device) + sd_model.unet.to(memory_format=torch.channels_last) + import torch._dynamo as dynamo # pylint: disable=unused-import + # torch._dynamo.config.log_level = logging.WARNING if shared.opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access + torch._dynamo.config.verbose = shared.opts.cuda_compile_verbose # pylint: disable=protected-access + torch._dynamo.config.suppress_errors = shared.opts.cuda_compile_errors # pylint: disable=protected-access + sd_model.unet = torch.compile(sd_model.unet, mode=shared.opts.cuda_compile_mode, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init + shared.log.info(f"Compiling pipeline={sd_model.__class__.__name__} shape={8 * sd_model.unet.config.sample_size} mode={shared.opts.cuda_compile_mode}") + sd_model("dummy prompt") + shared.log.info("Complilation done.") + + sd_model.sd_checkpoint_info = checkpoint_info # pylint: disable=attribute-defined-outside-init + sd_model.sd_model_checkpoint = checkpoint_info.filename # pylint: disable=attribute-defined-outside-init + sd_model.sd_model_hash = checkpoint_info.hash # pylint: disable=attribute-defined-outside-init sd_model.to(devices.device) except Exception as e: shared.log.error("Failed to load diffusers model") @@ -528,6 +608,60 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No shared.log.info(f'Model load finished: {memory_stats()}') +class DiffusersTaskType(Enum): + TEXT_2_IMAGE = 1 + IMAGE_2_IMAGE = 2 + INPAINTING = 3 + +def set_diffuser_pipe(pipe, new_pipe_type): + wrapper_pipe = None + + sd_checkpoint_info = pipe.sd_checkpoint_info + sd_model_checkpoint = pipe.sd_model_checkpoint + sd_model_hash = pipe.sd_model_hash + + if pipe.__class__ == PriorPipeline: + wrapper_pipe = pipe + pipe = pipe.main + + pipe_name = pipe.__class__.__name__ + pipe_name = pipe_name.replace("Img2Img", "").replace("Inpaint", "") + if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: + new_pipe_cls_str = pipe_name + elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: + new_pipe_cls_str = pipe_name.replace("Pipeline", "Img2ImgPipeline") + elif new_pipe_type == DiffusersTaskType.INPAINTING: + new_pipe_cls_str = pipe_name.replace("Pipeline", "InpaintPipeline") + + new_pipe_cls = getattr(diffusers, new_pipe_cls_str) + + if pipe.__class__ == new_pipe_cls: + return + + new_pipe = new_pipe_cls(**pipe.components) + + if wrapper_pipe is not None: + wrapper_pipe.main = new_pipe + new_pipe = wrapper_pipe + + new_pipe.sd_checkpoint_info = sd_checkpoint_info + new_pipe.sd_model_checkpoint = sd_model_checkpoint + new_pipe.sd_model_hash = sd_model_hash + + shared.sd_model = new_pipe + shared.log.info(f"Pipeline class changed from {pipe.__class__.__name__} to {new_pipe_cls.__name__}") + + +def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType: + if pipe.__class__ == PriorPipeline: + pipe = pipe.main + + if "Img2Img" in pipe.__class__.__name__: + return DiffusersTaskType.IMAGE_2_IMAGE + elif "Inpaint" in pipe.__class__.__name__: + return DiffusersTaskType.INPAINTING + return DiffusersTaskType.TEXT_2_IMAGE + def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None): from modules import lowvram, sd_hijack @@ -684,6 +818,11 @@ def apply_token_merging(sd_model, token_merging_ratio): return if current_token_merging_ratio > 0: tomesd.remove_patch(sd_model) + + if sd_model.__class__ == PriorPipeline: + # token merging is not supported for PriorPipelines currently + return + if token_merging_ratio > 0: tomesd.apply_patch( sd_model, diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index eeda69d0b..6033f1bf4 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -1,4 +1,4 @@ -from modules import sd_samplers_compvis, sd_samplers_kdiffusion, sd_samplers_diffusors, shared +from modules import sd_samplers_compvis, sd_samplers_kdiffusion, sd_samplers_diffusers, shared from modules.sd_samplers_common import samples_to_image_grid, sample_to_image # pylint: disable=unused-import from modules.shared import backend, Backend @@ -9,7 +9,7 @@ if backend == Backend.ORIGINAL: ] else: all_samplers = [ - *sd_samplers_diffusors.samplers_data_diffusors, + *sd_samplers_diffusers.samplers_data_diffusers, ] all_samplers_map = {x.name: x for x in all_samplers} samplers = all_samplers diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py new file mode 100644 index 000000000..ebb988109 --- /dev/null +++ b/modules/sd_samplers_diffusers.py @@ -0,0 +1,37 @@ +from diffusers import ( + DDIMScheduler, + DDPMScheduler, + DEISMultistepScheduler, + DPMSolverMultistepScheduler, + DPMSolverSinglestepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + HeunDiscreteScheduler, + KDPM2DiscreteScheduler, + PNDMScheduler, + UniPCMultistepScheduler, +) +from modules import sd_samplers_common + +samplers_data_diffusers = [ + sd_samplers_common.SamplerData('UniPC', lambda model: DiffusionSampler('UniPC', UniPCMultistepScheduler, model), [], {}), + sd_samplers_common.SamplerData('DDIM', lambda model: DiffusionSampler('DDIM', DDIMScheduler, model), [], {}), + sd_samplers_common.SamplerData('DDPM', lambda model: DiffusionSampler('DDPM', DDPMScheduler, model), [], {}), + sd_samplers_common.SamplerData('DEIS', lambda model: DiffusionSampler('DEIS', DEISMultistepScheduler, model), [], {}), + sd_samplers_common.SamplerData('DPM++ 2M', lambda model: DiffusionSampler('DPM++ 2M', DPMSolverMultistepScheduler, model), [], {}), + sd_samplers_common.SamplerData('DPM++ 1S', lambda model: DiffusionSampler('DPM++ 1S', DPMSolverSinglestepScheduler, model), [], {}), + sd_samplers_common.SamplerData('DPM++ 2M SDE', lambda model: DiffusionSampler('DPM++ 2M SDE', DPMSolverMultistepScheduler, model, algorithm_type="sde-dpmsolver++"), [], {}), + sd_samplers_common.SamplerData('DPM++ 2M Karras', lambda model: DiffusionSampler('DPM++ 2M Karras', DPMSolverMultistepScheduler, model, use_karras_sigmas=True), [], {}), + sd_samplers_common.SamplerData('DPM++ 1S Karras', lambda model: DiffusionSampler('DPM++ 1S Karras', DPMSolverSinglestepScheduler, model, use_karras_sigmas=True), [], {}), + sd_samplers_common.SamplerData('DPM++ 2M SDE Karras', lambda model: DiffusionSampler('DPM++ 2M SDE Karras', DPMSolverMultistepScheduler, model, use_karras_sigmas=True, algorithm_type="sde-dpmsolver++"), [], {}), + sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), + sd_samplers_common.SamplerData('Euler a', lambda model: DiffusionSampler('Euler a', EulerAncestralDiscreteScheduler, model), [], {}), + sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), + sd_samplers_common.SamplerData('DPM2++ 2M', lambda model: DiffusionSampler('KDPM2', KDPM2DiscreteScheduler, model), [], {}), + sd_samplers_common.SamplerData('PNDM', lambda model: DiffusionSampler('PNDM', PNDMScheduler, model), [], {}), +] + +class DiffusionSampler: + def __init__(self, name, constructor, sd_model, **kwargs): + self.sampler = constructor.from_pretrained(sd_model, subfolder="scheduler", **kwargs) + self.sampler.name = name diff --git a/modules/shared.py b/modules/shared.py index 7d5b939d3..f9ce0a981 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -4,11 +4,14 @@ import time import json import datetime import urllib.request +from urllib.parse import urlparse from enum import Enum +import tempfile import gradio as gr import tqdm import requests -from modules import errors, ui_components, shared_items, cmd_args +import diffusers +from modules import errors, ui_components, shared_items, cmd_args, modelloader from modules.paths_internal import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611 import modules.interrogate import modules.memmon @@ -72,6 +75,11 @@ ui_reorder_categories = [ ] +def is_url(string): + parsed_url = urlparse(string) + return all([parsed_url.scheme, parsed_url.netloc]) + + class Backend(Enum): ORIGINAL = 1 DIFFUSERS = 2 @@ -185,7 +193,7 @@ state.server_start = time.time() class OptionInfo: - def __init__(self, default=None, label="", component=None, component_args=None, onchange=None, section=None, refresh=None, comment_before='', comment_after=''): + def __init__(self, default=None, label="", component=None, component_args=None, onchange=None, section=None, refresh=None, submit=None, comment_before='', comment_after=''): self.default = default self.label = label self.component = component @@ -195,6 +203,7 @@ class OptionInfo: self.refresh = refresh self.comment_before = comment_before # HTML text that will be added after label in UI self.comment_after = comment_after # HTML text that will be added before label in UI + self.submit = submit def link(self, label, uri): self.comment_before += f"[{label}]" @@ -223,9 +232,70 @@ def list_checkpoint_tiles(): import modules.sd_models # pylint: disable=W0621 return modules.sd_models.checkpoint_tiles() - default_checkpoint = list_checkpoint_tiles()[0] if len(list_checkpoint_tiles()) > 0 else "model.ckpt" +def load_diffusers_ckpt(model_repo: str): + cached_dir = modelloader.download_diffusers_model(model_repo) + print(f"Downloaded {cached_dir}") + return "" + +def load_diffusers_lora(lora_repo: str): + pipe = sys.modules[__name__].sd_model + + if lora_repo == "": + pipe._remove_text_encoder_monkey_patch() # pylint: disable=W0212 + proc_cls_name = next(iter(pipe.unet.attn_processors.values())).__class__.__name__ + non_lora_proc_cls = getattr(diffusers.models.attention_processor, proc_cls_name[len("LORA"):]) + pipe.unet.set_attn_processor(non_lora_proc_cls()) + print("Removed LoRA.") + return "" + elif is_url(lora_repo): + with tempfile.TemporaryDirectory() as temp_dir: + os.system(f"wget -P {temp_dir} {lora_repo}") + temp_file_path = os.path.join(temp_dir, lora_repo.split('/')[-1]) + pipe.load_lora_weights(temp_file_path) + + lora_repo = '/'.join(lora_repo.split('/')[-2:]) + + print(f"Loaded Civit.ai LoRA: {lora_repo}") + return f"{lora_repo} is loaded. Pass empty text field to remove LoRA or pass new LoRA id." + elif len(lora_repo.split('/')) == 2: + lora_dir = os.path.dirname(opts.data["diffusers_dir"]) + cache_dir = os.path.join(lora_dir, "Diffusers_LoRA") + pipe.load_lora_weights(lora_repo, cache_dir=cache_dir) + print(f"Loaded {lora_repo}") + return f"{lora_repo} is loaded. Pass empty text field to remove LoRA or pass new LoRA id." + else: + print(f"{lora_repo} is not a valid LoRA identifier.") + return "" + +def load_diffusers_text_inv(text_inv_repo: str): + pipe = sys.modules[__name__].sd_model + + if text_inv_repo == "": + pipe.tokenizer = pipe.tokenizer.__class__.from_pretrained(pipe.tokenizer.name_or_path) + pipe.text_encoder.resize_token_embeddings(len(pipe.tokenizer)) + print("Removed all textual inversions.") + return "" + elif is_url(text_inv_repo): + with tempfile.TemporaryDirectory() as temp_dir: + os.system(f"wget -P {temp_dir} {text_inv_repo}") + temp_file_path = os.path.join(temp_dir, text_inv_repo.split('/')[-1]) + pipe.load_textual_inversion(temp_file_path) + + text_inv_repo = '/'.join(text_inv_repo.split('/')[-2:]) + + print(f"Loaded Civit.ai Textual Inv: {text_inv_repo}") + elif len(text_inv_repo.split('/')) == 2: + text_inv_dir = os.path.dirname(opts.data["diffusers_dir"]) + cache_dir = os.path.join(text_inv_dir, "Diffusers_Text_Inv") + pipe.load_textual_inversion(text_inv_repo, cache_dir=cache_dir) + print(f"Loaded {text_inv_repo}") + + text_inv_tokens = pipe.tokenizer.added_tokens_encoder.keys() + text_inv_tokens = [t for t in text_inv_tokens if not (len(t.split("_")) > 1 and t.split("_")[-1].isdigit())] + + return f"{', '.join(text_inv_tokens)} loaded. Pass empty text field to remove all or add new textual inversion id." def refresh_checkpoints(): import modules.sd_models # pylint: disable=W0621 @@ -329,7 +399,8 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "cuda_allow_tf32": OptionInfo(True, "Allow TF32 math ops"), "cuda_allow_tf16_reduced": OptionInfo(True, "Allow TF16 reduced precision math ops"), "cuda_compile": OptionInfo(False, "Enable model compile (experimental)"), - "cuda_compile_mode": OptionInfo("none", "Model compile mode (experimental)", gr.Radio, lambda: {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex']}), + "cuda_compile_mode": OptionInfo("none", "Model compile mode (experimental)", gr.Radio, lambda: {"choices": ['none', 'inductor', 'reduce-overhead', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex']}), + "cuda_compile_fullgraph": OptionInfo(False, "Model compile fullgraph"), "cuda_compile_verbose": OptionInfo(False, "Model compile verbose mode"), "cuda_compile_errors": OptionInfo(True, "Model compile suppress errors"), "disable_gc": OptionInfo(False, "Disable Torch memory garbage collection"), @@ -449,8 +520,9 @@ options_templates.update(options_section(('live-preview', "Live Previews"), { "logmonitor_refresh_period": OptionInfo(5000, "Log view update period, in milliseconds", gr.Slider, {"minimum": 0, "maximum": 30000, "step": 25}), })) + options_templates.update(options_section(('sampler-params', "Sampler Settings"), { - "show_samplers": OptionInfo(["Euler a", "UniPC", "DDIM", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), + "show_samplers": OptionInfo(["Euler a", "UniPC", "DDIM", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras", "DEIS"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), "fallback_sampler": OptionInfo("Euler a", "Secondary sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), "force_latent_sampler": OptionInfo("None", "Force latent upscaler sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems"), diff --git a/modules/ui.py b/modules/ui.py index 3edab8f2b..2a27df235 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -971,6 +971,7 @@ def create_ui(): quicksettings_names = opts.quicksettings_list quicksettings_names = {x: i for i, x in enumerate(quicksettings_names) if x != 'quicksettings'} quicksettings_list = [] + previous_section = [] tab_item_keys = [] current_tab = None @@ -1146,7 +1147,6 @@ def webpath(fn): web_path = os.path.relpath(fn, script_path).replace('\\', '/') else: web_path = os.path.abspath(fn) - return f'file={web_path}?{os.path.getmtime(fn)}' diff --git a/modules/ui_models.py b/modules/ui_models.py index aa1c082f6..afa6dc702 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -161,3 +161,15 @@ def create_ui(): return model_data, txt model_list_btn.click(fn=list_models, inputs=[], outputs=[model_table, models_outcome]) + + with gr.Tab(label="HF Hub"): + """" + options_templates.update(options_section(('diffusers', "Diffusers"), { + "diffusers_ckpt_download": OptionInfo("", "HFHub Checkpoint download", gr.Textbox, {"placeholder": "e.g. runwayml/stable-diffusion-v1-5"}, submit=load_diffusers_ckpt), + "diffusers_lora_download": OptionInfo("", "HFHub LoRA download", gr.Textbox, {"placeholder": "e.g. pcuenq/pokemon-lora"}, submit=load_diffusers_lora), + "diffusers_text_inv_download": OptionInfo("", "HFHub Textual Inversion download", gr.Textbox, {"placeholder": "e.g. sd-concepts-library/midjourney-style"}, submit=load_diffusers_text_inv), + })) + """ + + with gr.Tab(label="CivitAI"): + pass diff --git a/requirements.txt b/requirements.txt index 0aeebdd1a..fced77fd2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,6 +43,7 @@ yapf scikit-image basicsr compel +antlr4-python3-runtime==4.9.3 typing-extensions==4.6.3 pydantic==1.10.9 requests==2.31.0 From cc685a872993cc4d8969876aaca98897c20abac2 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 2 Jul 2023 21:07:26 -0400 Subject: [PATCH 02/42] wip diffusers --- DIFFUSERS.md | 22 +++++++++------ cli/hf-search.py | 7 ++--- modules/cmd_args.py | 2 ++ modules/modelloader.py | 7 +++-- modules/sd_models.py | 2 +- modules/shared.py | 24 ++-------------- modules/ui_models.py | 64 ++++++++++++++++++++++++++++++++++++------ 7 files changed, 81 insertions(+), 47 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index 199623c4d..52616aa2d 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -23,27 +23,31 @@ to update repo, do not use `--upgrade` flag, use manual `git pull` instead ### Diffusers - sd 1.5 and sd 2.1 model - models can be downloaded from huggingface hub - but focus on default model for now and i'll add downloader soon -- lora, textual inversion - only loras/textual-inversions downloaded from huggingface hub are supported for now - i'll add standard safetensors soon +- model downloader: tabs -> models -> hf hub - txt2img, img2img, inpaint, outpaint, process +- hires fix, restore faces, etc? -### Experimental +### Experimental - don't test yet -- cuda model compile using `reduce overhead` model with and without `fullgraph` +- cuda model compile using `reduce overhead` model with or without `fullgraph` - kandinsky model ## Todo -- enable loading of safetensors models +- lora +- embedding +- safetensors models - cleanup logging -- search&download models from hfhub - controlnet extension - multidiffusion extension - sdxl model +## Limitations + +- extra networks +- controlnet +- multi-diffusion + ## Issues - TBD diff --git a/cli/hf-search.py b/cli/hf-search.py index bbb4a4f0f..ac97b6c26 100755 --- a/cli/hf-search.py +++ b/cli/hf-search.py @@ -11,9 +11,8 @@ if __name__ == "__main__": model_filter = hf.ModelFilter( model_name=keyword, task='text-to-image', - tags='stable-diffusion', - library=['diffusers', 'stable-diffusion'], + library=['diffusers'], ) res = hf_api.list_models(filter=model_filter, full=True, limit=50, sort="downloads", direction=-1) - models = [{ 'name': m.modelId, 'downloads': m.downloads, 'mtime': m.lastModified, 'url': f'https://huggingface.co/{m.modelId}' } for m in res] - print('Online', models) + models = [{ 'name': m.modelId, 'downloads': m.downloads, 'mtime': m.lastModified, 'url': f'https://huggingface.co/{m.modelId}', 'pipeline': m.pipeline_tag, 'tags': m.tags } for m in res] + print(models) diff --git a/modules/cmd_args.py b/modules/cmd_args.py index 3c0c2ee6c..7d85e014c 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -90,6 +90,8 @@ def compatibility_args(opts, args): group.add_argument("--sub-quad-chunk-threshold", help=argparse.SUPPRESS, default=opts.sub_quad_chunk_threshold) group.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir) group.add_argument("--lyco-dir", help=argparse.SUPPRESS, default=opts.lyco_dir) + group.add_argument("--embeddings-dir", help=argparse.SUPPRESS, default=opts.embeddings_dir) + group.add_argument("--hypernetwork-dir", help=argparse.SUPPRESS, default=opts.hypernetwork_dir) group.add_argument("--lyco-patch-lora", help=argparse.SUPPRESS, default=opts.lyco_patch_lora) group.add_argument("--lyco-debug", help=argparse.SUPPRESS, action='store_true', default=False) group.add_argument("--enable-console-prompts", help=argparse.SUPPRESS, action='store_true', default=False) diff --git a/modules/modelloader.py b/modules/modelloader.py index 799537240..758dc7e34 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -11,6 +11,7 @@ from modules.paths import script_path, models_path diffuser_repos = [] + def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config: Dict[str, str] = None): from diffusers import DiffusionPipeline import huggingface_hub as hf @@ -41,6 +42,7 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config return pipeline_dir + def load_diffusers_models(model_path: str, command_path: str = None): import huggingface_hub as hf places = [] @@ -74,10 +76,9 @@ def find_diffuser(name: str): filt = hf.ModelFilter( model_name=name, task='text-to-image', - tags='stable-diffusion', - library=['diffusers', 'stable-diffusion'], + library=['diffusers'], ) - models = list(api.list_models(filter=filt, full=True, limit=50, sort="downloads", direction=-1)) + models = list(api.list_models(filter=filt, full=True, limit=5, sort="downloads", direction=-1)) shared.log.debug(f'Searching diffusers models: {name} {len(models) > 0}') if len(models) > 0: return models[0].modelId diff --git a/modules/sd_models.py b/modules/sd_models.py index d237f8728..b72f0d7dd 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -14,9 +14,9 @@ import safetensors.torch from omegaconf import OmegaConf import tomesd from transformers import logging as transformers_logging -import diffusers import ldm.modules.midas as midas from ldm.util import instantiate_from_config +import diffusers from modules import paths, shared, modelloader, devices, script_callbacks, sd_vae, sd_disable_initialization, errors, hashes, sd_models_config from modules.sd_hijack_inpainting import do_inpainting_hijack from modules.timer import Timer diff --git a/modules/shared.py b/modules/shared.py index f9ce0a981..3b31df4d2 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -234,31 +234,15 @@ def list_checkpoint_tiles(): default_checkpoint = list_checkpoint_tiles()[0] if len(list_checkpoint_tiles()) > 0 else "model.ckpt" -def load_diffusers_ckpt(model_repo: str): - cached_dir = modelloader.download_diffusers_model(model_repo) - print(f"Downloaded {cached_dir}") - return "" def load_diffusers_lora(lora_repo: str): pipe = sys.modules[__name__].sd_model - if lora_repo == "": pipe._remove_text_encoder_monkey_patch() # pylint: disable=W0212 proc_cls_name = next(iter(pipe.unet.attn_processors.values())).__class__.__name__ non_lora_proc_cls = getattr(diffusers.models.attention_processor, proc_cls_name[len("LORA"):]) pipe.unet.set_attn_processor(non_lora_proc_cls()) - print("Removed LoRA.") return "" - elif is_url(lora_repo): - with tempfile.TemporaryDirectory() as temp_dir: - os.system(f"wget -P {temp_dir} {lora_repo}") - temp_file_path = os.path.join(temp_dir, lora_repo.split('/')[-1]) - pipe.load_lora_weights(temp_file_path) - - lora_repo = '/'.join(lora_repo.split('/')[-2:]) - - print(f"Loaded Civit.ai LoRA: {lora_repo}") - return f"{lora_repo} is loaded. Pass empty text field to remove LoRA or pass new LoRA id." elif len(lora_repo.split('/')) == 2: lora_dir = os.path.dirname(opts.data["diffusers_dir"]) cache_dir = os.path.join(lora_dir, "Diffusers_LoRA") @@ -269,34 +253,30 @@ def load_diffusers_lora(lora_repo: str): print(f"{lora_repo} is not a valid LoRA identifier.") return "" + def load_diffusers_text_inv(text_inv_repo: str): pipe = sys.modules[__name__].sd_model - if text_inv_repo == "": pipe.tokenizer = pipe.tokenizer.__class__.from_pretrained(pipe.tokenizer.name_or_path) pipe.text_encoder.resize_token_embeddings(len(pipe.tokenizer)) - print("Removed all textual inversions.") return "" elif is_url(text_inv_repo): with tempfile.TemporaryDirectory() as temp_dir: os.system(f"wget -P {temp_dir} {text_inv_repo}") temp_file_path = os.path.join(temp_dir, text_inv_repo.split('/')[-1]) pipe.load_textual_inversion(temp_file_path) - text_inv_repo = '/'.join(text_inv_repo.split('/')[-2:]) - print(f"Loaded Civit.ai Textual Inv: {text_inv_repo}") elif len(text_inv_repo.split('/')) == 2: text_inv_dir = os.path.dirname(opts.data["diffusers_dir"]) cache_dir = os.path.join(text_inv_dir, "Diffusers_Text_Inv") pipe.load_textual_inversion(text_inv_repo, cache_dir=cache_dir) print(f"Loaded {text_inv_repo}") - text_inv_tokens = pipe.tokenizer.added_tokens_encoder.keys() text_inv_tokens = [t for t in text_inv_tokens if not (len(t.split("_")) > 1 and t.split("_")[-1].isdigit())] - return f"{', '.join(text_inv_tokens)} loaded. Pass empty text field to remove all or add new textual inversion id." + def refresh_checkpoints(): import modules.sd_models # pylint: disable=W0621 return modules.sd_models.list_models() diff --git a/modules/ui_models.py b/modules/ui_models.py index afa6dc702..369a3baec 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -162,14 +162,62 @@ def create_ui(): model_list_btn.click(fn=list_models, inputs=[], outputs=[model_table, models_outcome]) - with gr.Tab(label="HF Hub"): - """" - options_templates.update(options_section(('diffusers', "Diffusers"), { - "diffusers_ckpt_download": OptionInfo("", "HFHub Checkpoint download", gr.Textbox, {"placeholder": "e.g. runwayml/stable-diffusion-v1-5"}, submit=load_diffusers_ckpt), - "diffusers_lora_download": OptionInfo("", "HFHub LoRA download", gr.Textbox, {"placeholder": "e.g. pcuenq/pokemon-lora"}, submit=load_diffusers_lora), - "diffusers_text_inv_download": OptionInfo("", "HFHub Textual Inversion download", gr.Textbox, {"placeholder": "e.g. sd-concepts-library/midjourney-style"}, submit=load_diffusers_text_inv), - })) - """ + with gr.Tab(label="Huggingface"): + data = [] + os.environ.setdefault('HF_HUB_DISABLE_EXPERIMENTAL_WARNING', '1') + os.environ.setdefault('HF_HUB_DISABLE_SYMLINKS_WARNING', '1') + os.environ.setdefault('HF_HUB_DISABLE_IMPLICIT_TOKEN', '1') + os.environ.setdefault('HUGGINGFACE_HUB_VERBOSITY', 'warning') + + def hf_search(keyword): + import huggingface_hub as hf + hf_api = hf.HfApi() + model_filter = hf.ModelFilter( + model_name=keyword, + task='text-to-image', + library=['diffusers'], + ) + models = hf_api.list_models(filter=model_filter, full=True, limit=50, sort="downloads", direction=-1) + data.clear() + for model in models: + tags = [t for t in model.tags if not t.startswith('diffusers') and not t.startswith('license') and not t.startswith('arxiv') and len(t) > 2] + data.append([model.modelId, model.pipeline_tag, tags, model.downloads, model.lastModified, f'https://huggingface.co/{model.modelId}']) + return data + + def hf_select(evt: gr.SelectData): + return data[evt.index[0]][0] + + def hf_download_model(hub_id: str): + from modules.shared import log, opts + from modules.modelloader import download_diffusers_model + try: + download_diffusers_model(hub_id, cache_dir=opts.diffusers_dir) + except Exception as e: + log.error(f"Diffuser model downloaded error: model={hub_id} {e}") + return f"Diffuser model downloaded error: model={hub_id} {e}" + from modules.sd_models import list_models # pylint: disable=W0621 + list_models() + log.info(f"Diffuser model downloaded: model={hub_id}") + return f'Diffuser model downloaded: model={hub_id}' + + with gr.Row(): + hf_search_text = gr.Textbox('', label = 'Seach models', placeholder='search huggingface models') + + with gr.Row(): + hf_selected = gr.Textbox('', label = 'Select model', placeholder='select model from search results or enter model name manually') + with gr.Row(): + hf_download_model_btn = gr.Button(value="Download model", variant='primary') + + with gr.Row(): + hf_headers = ['Name', 'Pipeline', 'Tags', 'Downloads', 'Updated', 'URL'] + hf_results = gr.DataFrame([], label = 'Search results', show_label = True, interactive = False, wrap = True, overflow_row_behaviour = 'paginate', max_rows = 10, headers = hf_headers, type='array') + + hf_search_text.submit(fn=hf_search, inputs=[hf_search_text], outputs=[hf_results]) + hf_results.select(hf_select, inputs=None, outputs=[hf_selected]) + hf_download_model_btn.click(fn=hf_download_model, inputs=[hf_selected], outputs=[models_outcome]) + + # TODO load_diffusers_lora + # TODO load_diffusers_text_inv with gr.Tab(label="CivitAI"): pass From 8241e33868e945c73a55836fb3790e2181131034 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 3 Jul 2023 16:48:03 -0400 Subject: [PATCH 03/42] major diffusers update --- CHANGELOG.md | 5 ++ DIFFUSERS.md | 77 +++++++++++++++---- TODO.md | 1 + extensions-builtin/LDSR/ldsr_model_arch.py | 13 ++-- .../Lora/extra_networks_lora.py | 2 +- extensions-builtin/Lora/lora.py | 19 +++-- javascript/extraNetworks.js | 1 + javascript/style.css | 2 +- modules/cmd_args.py | 2 +- modules/lora_diffusers.py | 34 ++++++++ modules/modelloader.py | 20 ++--- modules/processing.py | 10 ++- modules/sd_hijack.py | 2 - modules/sd_models.py | 46 ++++++----- modules/sd_samplers_kdiffusion.py | 2 +- modules/shared.py | 62 ++------------- .../textual_inversion/textual_inversion.py | 48 +++++++----- modules/ui.py | 18 ++++- modules/ui_extra_networks.py | 24 +++--- modules/ui_extra_networks_checkpoints.py | 4 +- modules/ui_extra_networks_hypernets.py | 2 +- modules/ui_models.py | 21 ++--- webui.py | 2 +- 23 files changed, 247 insertions(+), 170 deletions(-) create mode 100644 modules/lora_diffusers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e60a29277..91ef21dc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log for SD.Next +## Update for ... + +- add settings -> extra networks -> do not automatically build extra network pages + speeds up app start if you have a lot of extra networks and you want to build them manually when needed + ## Update for 07/01/2023 Small quality-of-life updates and bugfixes: diff --git a/DIFFUSERS.md b/DIFFUSERS.md index 52616aa2d..e43898400 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -9,6 +9,7 @@ initial support merged into `dev` branch default sd 1.5 model will be downloaded automatically to `models/Diffusers` on first startup, disable **controlnet** and **multi-diffusion** extensions as right now they are not compatible with diffusers +lora support is not compatible with setting `Use LyCoris handler for all Lora types`, make sure its disabled to update repo, do not use `--upgrade` flag, use manual `git pull` instead @@ -16,38 +17,80 @@ to update repo, do not use `--upgrade` flag, use manual `git pull` instead ### Standard +goal is to test standard workflows (so not diffusers) to ensure there are no regressions +so diffusers code can be merged into `master` and we can continue with development there + - run with `webui --debug --backend original` -- goal is to test standard workflows (so not diffusers) to ensure there are no regressions - so diffusers code can be merged into `master` and we can continue with development there ### Diffusers -- sd 1.5 and sd 2.1 model -- model downloader: tabs -> models -> hf hub -- txt2img, img2img, inpaint, outpaint, process -- hires fix, restore faces, etc? +whats implemented so far? -### Experimental - don't test yet +- simple model downloader for huggingface models: tabs -> models -> hf hub +- use huggingface models +- extra networks ui +- use safetensor models with diffusers backend +- standard workflows: + - txt2img, img2img, inpaint, outpaint, process + - hires fix, restore faces, etc? +- textual inversion + yes, this applies to standard embedddings, don't need ones from huggingface +- lora + yes, this applies to standard loras, don't need ones from huggingface + but seems that diffuser lora support is somewhat limited, so quite a few loras may not work + you should see which lora loads without issues in console log +- system info tab with updated information +- kandinsky model + works for me -- cuda model compile using `reduce overhead` model with or without `fullgraph` -- kandinsky model +### Experimental + +- cuda model compile + in settings -> compute settings + diffusers recommend `reduce overhead`, but other methods are available as well + it seems that fullgraph is possible (with sufficient vram) when using diffusers +- deepfloyd + in theory it should work, but its 20gb model so cant test it just yet + note that access is gated, so you'll need to download using your huggingface credentials + (you can still do it from sdnext ui, just need access token) ## Todo -- lora -- embedding -- safetensors models -- cleanup logging -- controlnet extension -- multidiffusion extension - sdxl model ## Limitations -- extra networks +even if extensions are not supported, runtime errors are never nice +will need to handle in the code before we get out of alpha + - controlnet -- multi-diffusion + `sd_model.model?.diffusion_model?` +- multi-diffusion + `sd_model.first_stage_model?.encoder?` +- lycoris + `lyco_patch_lora` ## Issues - TBD + +## Notes for HF + +- removed `quicksettings` alternative completely +- added simple model downloader in ui: *tabs -> models -> huggingface* +- redone **textual inversion** support, core is now in `modules/textual_inversion/textual_inversion.py:load_diffusers_embedding()` + the point is that sdnext pre-loads all compatible embeddings on model load so they are available in prompt context +- added support for diffuser models in **safetensors/ckpt** format + btw, when i use: `diffusers.StableDiffusionPipeline.from_ckpt` + first time it downloads something - what is that? + > Downloading (…)lve/main/config.json: 4.55k + > Downloading pytorch_model.bin: 1.22G + and in general, loading safetensors model is quite slow, is that expected? + for example, 2sec vs 18sec +- in `modules/modelloader.py:download_diffusers_model()` i get unknown property for `hf.model_info(hub_id).cardData` + can you double-check if this is linter issue or actual problem? +- redone **lora** support, core is now in `modules/lora_diffusers.py` +- question on `pipe.load_lora_weights` + does it support loading multiple loras? i don't see any notes on that in docs + also, lora strength is specified using `cross_attention_kwargs={"scale": x}` during pipeline execution + which means if there are multiple loras, they all have the same strength? diff --git a/TODO.md b/TODO.md index 19ce7d49f..ef0e08985 100644 --- a/TODO.md +++ b/TODO.md @@ -41,6 +41,7 @@ Tech that can be integrated as part of the core workflow... - [DataComp CLiP](https://github.com/mlfoundations/open_clip/blob/main/docs/datacomp_models.md) - [ClipSeg](https://github.com/timojl/clipseg) - [DragGAN](https://github.com/XingangPan/DragGAN) +- [LamaCleaner]([Title](https://github.com/Sanster/lama-cleaner)) - `TensorRT` ## Random diff --git a/extensions-builtin/LDSR/ldsr_model_arch.py b/extensions-builtin/LDSR/ldsr_model_arch.py index 41d97d071..9411e7374 100644 --- a/extensions-builtin/LDSR/ldsr_model_arch.py +++ b/extensions-builtin/LDSR/ldsr_model_arch.py @@ -23,10 +23,10 @@ class LDSR: global cached_ldsr_model if shared.opts.ldsr_cached and cached_ldsr_model is not None: - print("Loading model from cache") + shared.log.info("LDSR Loading model from cache") model: torch.nn.Module = cached_ldsr_model else: - print(f"Loading model from {self.modelPath}") + shared.log.info(f"LDSR Loading model from {self.modelPath}") _, extension = os.path.splitext(self.modelPath) if extension.lower() == ".safetensors": pl_sd = safetensors.torch.load_file(self.modelPath, device="cpu") @@ -126,11 +126,10 @@ class LDSR: height_downsampled_pre = int(np.ceil(hd)) if down_sample_rate != 1: - print( - f'Downsampling from [{width_og}, {height_og}] to [{width_downsampled_pre}, {height_downsampled_pre}]') + shared.log.info(f'LDSR Downsampling from [{width_og}, {height_og}] to [{width_downsampled_pre}, {height_downsampled_pre}]') im_og = im_og.resize((width_downsampled_pre, height_downsampled_pre), Image.LANCZOS) else: - print(f"Down sample rate is 1 from {target_scale} / 4 (Not downsampling)") + shared.log.info(f"LDSR Downsample rate is 1 from {target_scale} / 4 (Not downsampling)") # pad width and height to multiples of 64, pads with the edge values of image to avoid artifacts pad_w, pad_h = np.max(((2, 2), np.ceil(np.array(im_og.size) / 64).astype(int)), axis=0) * 64 - im_og.size @@ -183,7 +182,7 @@ def convsample_ddim(model, cond, steps, shape, eta=1.0, callback=None, normals_s ddim = DDIMSampler(model) bs = shape[0] shape = shape[1:] - print(f"Sampling with eta = {eta}; steps: {steps}") + shared.log.info(f"LDSR Sampling with eta = {eta}; steps: {steps}") samples, intermediates = ddim.sample(steps, batch_size=bs, shape=shape, conditioning=cond, callback=callback, normals_sequence=normals_sequence, quantize_x0=quantize_x0, eta=eta, mask=mask, x0=x0, temperature=temperature, verbose=False, @@ -206,7 +205,7 @@ def make_convolutional_sample(batch, model, custom_steps=None, eta=1.0, quantize if custom_shape is not None: z = torch.randn(custom_shape) - print(f"Generating {custom_shape[0]} samples of shape {custom_shape[1:]}") + shared.log.info(f"LDSR Generating {custom_shape[0]} samples of shape {custom_shape[1:]}") z0 = None diff --git a/extensions-builtin/Lora/extra_networks_lora.py b/extensions-builtin/Lora/extra_networks_lora.py index b5fea4d2e..bee0477ed 100644 --- a/extensions-builtin/Lora/extra_networks_lora.py +++ b/extensions-builtin/Lora/extra_networks_lora.py @@ -1,5 +1,5 @@ -from modules import extra_networks, shared import lora +from modules import extra_networks, shared class ExtraNetworkLora(extra_networks.ExtraNetwork): diff --git a/extensions-builtin/Lora/lora.py b/extensions-builtin/Lora/lora.py index 5273c9f82..088be31c1 100644 --- a/extensions-builtin/Lora/lora.py +++ b/extensions-builtin/Lora/lora.py @@ -1,8 +1,7 @@ import os import re -import torch from typing import Union - +import torch from modules import shared, devices, sd_models, errors, scripts, sd_hijack, hashes metadata_tags_order = {"ss_sd_model_name": 1, "ss_resolution": 2, "ss_clip_skip": 3, "ss_num_train_images": 10, "ss_tag_frequency": 20} @@ -127,7 +126,6 @@ class LoraModule: self.multiplier = 1.0 self.modules = {} self.mtime = None - self.mentioned_name = None """the text that was used to add lora to prompt - can be either name or an alias""" @@ -154,6 +152,13 @@ def assign_lora_names_to_compvis_modules(sd_model): sd_model.lora_layer_mapping = lora_layer_mapping +def load_diffuser_lora(name, lora_on_disk, multiplier): + lora = LoraModule(name, lora_on_disk) + lora.mtime = os.path.getmtime(lora_on_disk.filename) + from modules.lora_diffusers import load_diffusers_lora + load_diffusers_lora(name, lora_on_disk, multiplier) + return lora + def load_lora(name, lora_on_disk): lora = LoraModule(name, lora_on_disk) @@ -205,7 +210,6 @@ def load_lora(name, lora_on_disk): else: print(f'Lora layer {key_diffusers} matched a layer with unsupported type: {type(sd_module).__name__}') continue - raise AssertionError(f"Lora layer {key_diffusers} matched a layer with unsupported type: {type(sd_module).__name__}") with torch.no_grad(): module.weight.copy_(weight) @@ -243,14 +247,17 @@ def load_loras(names, multipliers=None): failed_to_load_loras = [] for i, name in enumerate(names): - lora = already_loaded.get(name, None) + lora = already_loaded.get(name, None) if shared.backend == shared.Backend.ORIGINAL else None lora_on_disk = loras_on_disk[i] if lora_on_disk is not None: if lora is None or os.path.getmtime(lora_on_disk.filename) > lora.mtime: try: - lora = load_lora(name, lora_on_disk) + if shared.backend == shared.Backend.DIFFUSERS: + lora = load_diffuser_lora(name, lora_on_disk, multipliers[i] if multipliers else 1.0) + else: + lora = load_lora(name, lora_on_disk) except Exception as e: errors.display(e, f"loading Lora {lora_on_disk.filename}") continue diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index f2e763362..39f75c91c 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -19,6 +19,7 @@ function setupExtraNetworksForTab(tabname) { searchTerm = search.value.toLowerCase(); gradioApp().querySelectorAll(`#${tabname}_extra_tabs div.card`).forEach((elem) => { text = `${elem.querySelector('.name').textContent.toLowerCase()} ${elem.querySelector('.search_term').textContent.toLowerCase()}`; + text = text.replace('models--', 'Diffusers') elem.style.display = text.indexOf(searchTerm) == -1 ? 'none' : ''; }); }); diff --git a/javascript/style.css b/javascript/style.css index 19faaa021..eef8d75ee 100644 --- a/javascript/style.css +++ b/javascript/style.css @@ -540,7 +540,7 @@ table.settings-value-table td{ .extra-networks-tab { padding: 0 !important; } .extra-network-subdirs { background: var(--input-background-fill); } .extra-networks-page { display: flex } -.extra-networks .custom-button { min-width: 60px; width: 100%; background: none; justify-content: left; padding: 2px 8px 2px 8px; box-shadow: none; } +.extra-networks .custom-button { min-width: 80px; max-width: 240px; width: 100%; background: none; justify-content: left; text-align: left; padding: 2px 8px 2px 8px; box-shadow: none; } .extra-networks .custom-button:hover { background: var(--button-primary-background-fill) } .extra-network-cards { display: flex; flex-wrap: wrap; height: 50vh; overflow-y: scroll; overflow-x: hidden; scroll-snap-type: y mandatory; width: -webkit-fill-available; } .extra-network-cards .card { height: fit-content; margin: 0.5em; position: relative; scroll-snap-align: start; scroll-margin-top: 0; } diff --git a/modules/cmd_args.py b/modules/cmd_args.py index 7d85e014c..fe37814e9 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -45,7 +45,7 @@ group.add_argument('--use-directml', default = False, action='store_true', help group.add_argument("--use-cuda", default=False, action='store_true', help="Force use nVidia CUDA backend, default: %(default)s") group.add_argument("--use-rocm", default=False, action='store_true', help="Force use AMD ROCm backend, default: %(default)s") group.add_argument('--subpath', type=str, help='Customize the URL subpath for usage with reverse proxy') -group.add_argument('--backend', type=str, choices=[None, 'original', 'diffusers'], default=None, required=False, help='force backend type') +group.add_argument('--backend', type=str, choices=['original', 'diffusers'], required=False, help='force model pipeline type') # removed args are added here as hidden in fixed format for compatbility reasons diff --git a/modules/lora_diffusers.py b/modules/lora_diffusers.py new file mode 100644 index 000000000..a6a5cbad7 --- /dev/null +++ b/modules/lora_diffusers.py @@ -0,0 +1,34 @@ +import diffusers +from modules import shared + +lora_state = { # TODO this is ugly but diffusers + 'multiplier': 1.0, + 'active': False, + 'loaded': 0, +} + +def unload_diffusers_lora(): + try: + pipe = shared.sd_model + lora_state['active'] = False + lora_state['loaded'] = 0 + pipe._remove_text_encoder_monkey_patch() # pylint: disable=W0212 + proc_cls_name = next(iter(pipe.unet.attn_processors.values())).__class__.__name__ + non_lora_proc_cls = getattr(diffusers.models.attention_processor, proc_cls_name[len("LORA"):]) + pipe.unet.set_attn_processor(non_lora_proc_cls()) + # shared.log.debug('Diffusers LoRA unloaded') + except Exception: + pass + + +def load_diffusers_lora(name, lora, strength = 1.0): + try: + pipe = shared.sd_model + pipe.load_lora_weights(lora.filename, cache_dir=shared.opts.diffusers_dir, local_files_only=True) + lora_state['active'] = True + lora_state['loaded'] += 1 + lora_state['multiplier'] = strength + # pipe.unet.load_attn_procs("pcuenq/pokemon-lora") + shared.log.info(f"Diffusers LoRA loaded: {name} {lora_state['multiplier']}") + except Exception as e: + shared.log.error(f"Diffusers LoRA loading failed: {name} {e}") diff --git a/modules/modelloader.py b/modules/modelloader.py index 758dc7e34..6de1efe6a 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -12,7 +12,7 @@ from modules.paths import script_path, models_path diffuser_repos = [] -def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config: Dict[str, str] = None): +def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config: Dict[str, str] = None, token = None): from diffusers import DiffusionPipeline import huggingface_hub as hf @@ -21,14 +21,16 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config "force_download": False, "resume_download": True, "cache_dir": shared.opts.diffusers_dir, + # "use_auth_token": True, } - if cache_dir is not None: download_config["cache_dir"] = cache_dir - + shared.log.debug(f"Diffusers downloading: {hub_id} to {cache_dir}") + if token is not None and len(token) > 2: + shared.log.debug(f"Diffusers authentication: {token}") + hf.login(token) pipeline_dir = DiffusionPipeline.download(hub_id, **download_config) model_info_dict = hf.model_info(hub_id).cardData # TODO hfhub card-data? - # some checkpoints need to be downloaded as "hidden" as they just serve as pre- or post-pipelines of other pipelines if model_info_dict is not None and "prior" in model_info_dict: download_dir = DiffusionPipeline.download(model_info_dict["prior"], **download_config) @@ -36,10 +38,8 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config # mark prior as hidden with open(os.path.join(download_dir, "hidden"), "w", encoding="utf-8") as f: f.write("True") - with open(os.path.join(pipeline_dir, "model_info.json"), "w", encoding="utf-8") as json_file: json.dump(model_info_dict, json_file) - return pipeline_dir @@ -61,7 +61,7 @@ def load_diffusers_models(model_path: str, command_path: str = None): output.append(str(r.repo_id)) except Exception as e: shared.log.error(f"Error listing diffusers: {place} {e}") - shared.log.debug(f'Scanning diffusers cache: {len(output)} {model_path} {command_path}') + shared.log.debug(f'Scanning diffusers cache: {model_path} {command_path} {len(output)}') return output @@ -105,7 +105,7 @@ def load_models(model_path: str, model_url: str = None, command_path: str = None for place in places: for full_path in shared.walk_files(place, allowed_extensions=ext_filter): if os.path.islink(full_path) and not os.path.exists(full_path): - print(f"Skipping broken symlink: {full_path}") + shared.log.error(f"Skipping broken symlink: {full_path}") continue if ext_blacklist is not None and any(full_path.endswith(x) for x in ext_blacklist): continue @@ -172,13 +172,13 @@ def move_files(src_path: str, dest_path: str, ext_filter: str = None): if ext_filter is not None: if ext_filter not in file: continue - print(f"Moving {file} from {src_path} to {dest_path}.") + shared.log.warning(f"Moving {file} from {src_path} to {dest_path}.") try: shutil.move(fullpath, dest_path) except Exception: pass if len(os.listdir(src_path)) == 0: - print(f"Removing empty folder: {src_path}") + shared.log.info(f"Removing empty folder: {src_path}") shutil.rmtree(src_path, True) except Exception: pass diff --git a/modules/processing.py b/modules/processing.py index e8808a910..68c18c977 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -26,6 +26,7 @@ import modules.images as images import modules.styles import modules.sd_models as sd_models import modules.sd_vae as sd_vae +from modules.lora_diffusers import lora_state, unload_diffusers_lora opt_C = 4 @@ -697,6 +698,10 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: # TODO(Patrick): For wrapped pipelines this is currently a no-op shared.sd_model.scheduler = scheduler.sampler + cross_attention_kwargs={} + if lora_state['active']: + cross_attention_kwargs['scale'] = lora_state['multiplier'] + task_specific_kwargs={} if sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE: task_specific_kwargs = {"height": p.height, "width": p.width} elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE: @@ -704,7 +709,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING: # TODO(PVP): change out to latents once possible with `diffusers` task_specific_kwargs = {"image": p.init_images[0], "mask_image": p.image_mask, "strength": p.denoising_strength} - output = shared.sd_model( prompt=prompts, negative_prompt=negative_prompts, @@ -712,9 +716,13 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: guidance_scale=p.cfg_scale, generator=generator, output_type="np", + cross_attention_kwargs=cross_attention_kwargs, **task_specific_kwargs ) x_samples_ddim = output.images + if lora_state['active']: + unload_diffusers_lora() + else: raise ValueError(f"Unknown backend {backend}") diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 5e397ebb9..5464cee44 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -174,14 +174,12 @@ class StableDiffusionModelHijack: sd_hijack_unet.hijack_ddpm_edit() if opts.cuda_compile and opts.cuda_compile_mode == 'ipex': - import logging if shared.cmd_opts.use_ipex: shared.log.info("Model compile enabled: IPEX Optimize Graph Mode") else: shared.log.warning("Model compile skipped: IPEX Method is for Intel GPU's with OneAPI") elif opts.cuda_compile and opts.cuda_compile_mode != 'none' and shared.backend == shared.Backend.ORIGINAL: try: - import logging import torch._dynamo as dynamo # pylint: disable=unused-import # torch._dynamo.config.log_level = logging.WARNING if opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access torch._dynamo.config.verbose = opts.cuda_compile_verbose # pylint: disable=protected-access diff --git a/modules/sd_models.py b/modules/sd_models.py index b72f0d7dd..2181b09d7 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -41,8 +41,10 @@ class CheckpointInfo: self.name = None self.hash = None self.filename = filename + self.type = '' abspath = os.path.abspath(filename) - if shared.backend == shared.Backend.ORIGINAL: + + if os.path.isfile(abspath): # ckpt or safetensor if shared.opts.ckpt_dir is not None and abspath.startswith(shared.opts.ckpt_dir): name = abspath.replace(shared.opts.ckpt_dir, '') elif abspath.startswith(model_path): @@ -54,7 +56,9 @@ class CheckpointInfo: self.name = name self.hash = model_hash(self.filename) self.sha256 = hashes.sha256_from_cache(self.filename, f"checkpoint/{name}") - elif shared.backend == shared.Backend.DIFFUSERS: + self.path = abspath + self.type = abspath.split('.')[-1].lower() + else: # maybe a diffuser repo = [r for r in modelloader.diffuser_repos if filename == r['filename']] if len(repo) == 0: error_message = f'Cannot find diffuser model: {filename}' @@ -64,6 +68,7 @@ class CheckpointInfo: self.hash = repo[0]['hash'][:8] self.sha256 = repo[0]['hash'] self.path = repo[0]['path'] + self.type = 'diffusers' if os.path.isfile(repo[0]['model_info']): file_path = repo[0]['model_info'] @@ -71,8 +76,6 @@ class CheckpointInfo: self.model_info = json.load(json_file) else: self.model_info = None - else: - raise ValueError(f'Unknown backend: {shared.backend}') self.name_for_extra = os.path.splitext(os.path.basename(filename))[0] self.model_name = os.path.splitext(name.replace("/", "_").replace("\\", "_"))[0] @@ -123,13 +126,10 @@ def checkpoint_tiles(): def list_models(): checkpoints_list.clear() checkpoint_aliases.clear() - if shared.backend == shared.Backend.ORIGINAL: - ext_filter=[".safetensors"] if shared.opts.sd_disable_ckpt else [".ckpt", ".safetensors"] - model_list = modelloader.load_models(model_path=os.path.join(models_path, 'Stable-diffusion'), model_url=None, command_path=shared.opts.ckpt_dir, ext_filter=ext_filter, download_name=None, ext_blacklist=[".vae.ckpt", ".vae.safetensors"]) - else: - global model_path # pylint: disable=global-statement - model_path = os.path.join(models_path, 'Diffusers') - model_list = modelloader.load_diffusers_models(model_path=model_path, command_path=shared.opts.diffusers_dir) + ext_filter=[".safetensors"] if shared.opts.sd_disable_ckpt else [".ckpt", ".safetensors"] + model_list = modelloader.load_models(model_path=model_path, model_url=None, command_path=shared.opts.ckpt_dir, ext_filter=ext_filter, download_name=None, ext_blacklist=[".vae.ckpt", ".vae.safetensors"]) + if shared.backend == shared.Backend.DIFFUSERS: + model_list += modelloader.load_diffusers_models(model_path=os.path.join(models_path, 'Diffusers'), command_path=shared.opts.diffusers_dir) for filename in sorted(model_list, key=str.lower): checkpoint_info = CheckpointInfo(filename) @@ -158,8 +158,8 @@ def list_models(): model_list = modelloader.load_models(model_path=model_path, model_url=model_url, command_path=shared.opts.ckpt_dir, ext_filter=[".ckpt", ".safetensors"], download_name="v1-5-pruned-emaonly.safetensors", ext_blacklist=[".vae.ckpt", ".vae.safetensors"]) else: default_model_id = "runwayml/stable-diffusion-v1-5" - modelloader.download_diffusers_model(default_model_id, model_path) - model_list = modelloader.load_diffusers_models(model_path=model_path, command_path=shared.opts.diffusers_dir) + modelloader.download_diffusers_model(default_model_id, os.path.join(models_path, 'Diffusers')) + model_list = modelloader.load_diffusers_models(model_path=os.path.join(models_path, 'Diffusers'), command_path=shared.opts.diffusers_dir) for filename in sorted(model_list, key=str.lower): checkpoint_info = CheckpointInfo(filename) @@ -549,19 +549,22 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No try: if shared.cmd_opts.ckpt is not None and model_data.initial: # initial load model_name = modelloader.find_diffuser(shared.cmd_opts.ckpt) - if model_name is not None: shared.log.info(f'Loading diffuser model: {model_name}') model_file = modelloader.download_diffusers_model(hub_id=model_name) sd_model = diffusers.DiffusionPipeline.from_pretrained(model_file, **diffusers_load_config) - list_models() # rescan for downloaded model checkpoint_info = CheckpointInfo(model_name) if sd_model is None: checkpoint_info = checkpoint_info or select_checkpoint() shared.log.info(f'Loading diffuser model: {checkpoint_info.filename}') - sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) + if not os.path.isfile(checkpoint_info.path): + sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) + else: + diffusers_load_config["local_files_only "] = True + diffusers_load_config["extract_ema"] = True + sd_model = diffusers.StableDiffusionPipeline.from_ckpt(checkpoint_info.path, **diffusers_load_config) if "StableDiffusion" in sd_model.__class__.__name__: sd_model.scheduler = diffusers.UniPCMultistepScheduler.from_config(sd_model.scheduler.config) @@ -570,9 +573,9 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.scheduler.name = 'DDIM' # Prior pipelines - if checkpoint_info.model_info is not None and "prior" in checkpoint_info.model_info: + if hasattr(checkpoint_info, 'model_info') and checkpoint_info.model_info is not None and "prior" in checkpoint_info.model_info: prior_id = checkpoint_info.model_info["prior"] - shared.log.info(f"Loading prior {prior_id} for {checkpoint_info.filename}") + shared.log.info(f"Loading diffuser prior: {checkpoint_info.filename} {prior_id}") prior = diffusers.DiffusionPipeline.from_pretrained(prior_id, **diffusers_load_config) sd_model = PriorPipeline(prior=prior, main=sd_model) # wrap sd_model @@ -586,7 +589,6 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.to(devices.device) sd_model.unet.to(memory_format=torch.channels_last) import torch._dynamo as dynamo # pylint: disable=unused-import - # torch._dynamo.config.log_level = logging.WARNING if shared.opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access torch._dynamo.config.verbose = shared.opts.cuda_compile_verbose # pylint: disable=protected-access torch._dynamo.config.suppress_errors = shared.opts.cuda_compile_errors # pylint: disable=protected-access sd_model.unet = torch.compile(sd_model.unet, mode=shared.opts.cuda_compile_mode, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init @@ -602,6 +604,12 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No shared.log.error("Failed to load diffusers model") errors.display(e, "loading Diffusers model") shared.sd_model = sd_model + + from modules.textual_inversion import textual_inversion + embedding_db = textual_inversion.EmbeddingDatabase() + embedding_db.add_embedding_dir(shared.opts.embeddings_dir) + embedding_db.load_textual_inversion_embeddings(force_reload=True) + timer.record("load") shared.log.info(f"Model loaded in {timer.summary()}") devices.torch_gc(force=True) diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py index 2393b2ec2..f15d3c09b 100644 --- a/modules/sd_samplers_kdiffusion.py +++ b/modules/sd_samplers_kdiffusion.py @@ -330,7 +330,7 @@ class KDiffusionSampler: try: return BrownianTreeNoiseSampler(x.to("cpu"), sigma_min, sigma_max, seed=current_iter_seeds, transform=lambda x: x.to("cpu"), transform_last=lambda x: x.to(shared.device)) # pylint: disable=E1123 except Exception: - print("ERROR Please apply this patch to repositories/k-diffusion/k_diffusion/sampling.py: https://github.com/crowsonkb/k-diffusion/pull/68/files") + shared.log.error("Apply this patch to repositories/k-diffusion/k_diffusion/sampling.py: https://github.com/crowsonkb/k-diffusion/pull/68/files") return None else: return BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=current_iter_seeds) diff --git a/modules/shared.py b/modules/shared.py index 3b31df4d2..5f480f562 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -6,12 +6,10 @@ import datetime import urllib.request from urllib.parse import urlparse from enum import Enum -import tempfile import gradio as gr import tqdm import requests -import diffusers -from modules import errors, ui_components, shared_items, cmd_args, modelloader +from modules import errors, ui_components, shared_items, cmd_args from modules.paths_internal import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611 import modules.interrogate import modules.memmon @@ -235,48 +233,6 @@ def list_checkpoint_tiles(): default_checkpoint = list_checkpoint_tiles()[0] if len(list_checkpoint_tiles()) > 0 else "model.ckpt" -def load_diffusers_lora(lora_repo: str): - pipe = sys.modules[__name__].sd_model - if lora_repo == "": - pipe._remove_text_encoder_monkey_patch() # pylint: disable=W0212 - proc_cls_name = next(iter(pipe.unet.attn_processors.values())).__class__.__name__ - non_lora_proc_cls = getattr(diffusers.models.attention_processor, proc_cls_name[len("LORA"):]) - pipe.unet.set_attn_processor(non_lora_proc_cls()) - return "" - elif len(lora_repo.split('/')) == 2: - lora_dir = os.path.dirname(opts.data["diffusers_dir"]) - cache_dir = os.path.join(lora_dir, "Diffusers_LoRA") - pipe.load_lora_weights(lora_repo, cache_dir=cache_dir) - print(f"Loaded {lora_repo}") - return f"{lora_repo} is loaded. Pass empty text field to remove LoRA or pass new LoRA id." - else: - print(f"{lora_repo} is not a valid LoRA identifier.") - return "" - - -def load_diffusers_text_inv(text_inv_repo: str): - pipe = sys.modules[__name__].sd_model - if text_inv_repo == "": - pipe.tokenizer = pipe.tokenizer.__class__.from_pretrained(pipe.tokenizer.name_or_path) - pipe.text_encoder.resize_token_embeddings(len(pipe.tokenizer)) - return "" - elif is_url(text_inv_repo): - with tempfile.TemporaryDirectory() as temp_dir: - os.system(f"wget -P {temp_dir} {text_inv_repo}") - temp_file_path = os.path.join(temp_dir, text_inv_repo.split('/')[-1]) - pipe.load_textual_inversion(temp_file_path) - text_inv_repo = '/'.join(text_inv_repo.split('/')[-2:]) - print(f"Loaded Civit.ai Textual Inv: {text_inv_repo}") - elif len(text_inv_repo.split('/')) == 2: - text_inv_dir = os.path.dirname(opts.data["diffusers_dir"]) - cache_dir = os.path.join(text_inv_dir, "Diffusers_Text_Inv") - pipe.load_textual_inversion(text_inv_repo, cache_dir=cache_dir) - print(f"Loaded {text_inv_repo}") - text_inv_tokens = pipe.tokenizer.added_tokens_encoder.keys() - text_inv_tokens = [t for t in text_inv_tokens if not (len(t.split("_")) > 1 and t.split("_")[-1].isdigit())] - return f"{', '.join(text_inv_tokens)} loaded. Pass empty text field to remove all or add new textual inversion id." - - def refresh_checkpoints(): import modules.sd_models # pylint: disable=W0621 return modules.sd_models.list_models() @@ -349,7 +305,7 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), { "prompt_mean_norm": OptionInfo(True, "Prompt attention mean normalization"), "comma_padding_backtrack": OptionInfo(20, "Prompt padding for long prompts", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }), "sd_disable_ckpt": OptionInfo(False, "Disallow usage of checkpoints in ckpt format"), - "sd_backend": OptionInfo("Original", "Stable Diffusion backend (experimental)", gr.Radio, lambda: {"choices": ["Original", "Diffusers"] }), + "sd_backend": OptionInfo("original", "Stable Diffusion backend (experimental)", gr.Radio, lambda: {"choices": ["original", "diffusers"] }), })) options_templates.update(options_section(('optimizations', "Optimizations"), { @@ -483,7 +439,6 @@ options_templates.update(options_section(('ui', "User Interface"), { "ui_tab_reorder": OptionInfo("From Text, From Image, Process Image", "UI tabs order"), "ui_scripts_reorder": OptionInfo("Enable Dynamic Thresholding, ControlNet", "UI scripts order"), "ui_reorder": OptionInfo(", ".join(ui_reorder_categories), "txt2img/img2img UI item order"), - "ui_extra_networks_tab_reorder": OptionInfo("Checkpoints, Lora, LyCORIS, Textual Inversion, Hypernetworks", "Extra networks tab order"), })) options_templates.update(options_section(('live-preview', "Live Previews"), { @@ -573,11 +528,13 @@ options_templates.update(options_section(('upscaling', "Upscaling"), { })) options_templates.update(options_section(('extra_networks', "Extra Networks"), { + "ui_extra_networks_tab_reorder": OptionInfo("Checkpoints, Lora, LyCORIS, Textual Inversion, Hypernetworks", "Extra networks tab order"), "extra_networks_card_cover": OptionInfo("inline", "UI position", gr.Radio, lambda: {"choices": ["cover", "inline", "sidebar"]}), "extra_networks_sidebar_width": OptionInfo(35, "UI sidebar width (%)", gr.Slider, {"minimum": 10, "maximum": 80, "step": 1}), "extra_networks_card_size": OptionInfo(200, "UI card size (px)", gr.Slider, {"minimum": 20, "maximum": 2000, "step": 1}), "extra_networks_card_square": OptionInfo(False, "UI disable variable aspect ratio"), "extra_networks_card_fit": OptionInfo("cover", "UI image contain method", gr.Radio, lambda: {"choices": ["contain", "cover", "fill"]}), + "extra_network_skip_indexing": OptionInfo(False, "Do not automatically build extra network pages", gr.Checkbox), "lyco_patch_lora": OptionInfo(False, "Use LyCoris handler for all Lora types", gr.Checkbox), "lora_disable": OptionInfo(False, "Disable built-in Lora handler", gr.Checkbox, { "visible": True }, onchange=lora_disable), "lora_functional": OptionInfo(False, "Use Kohya method for handling multiple Loras", gr.Checkbox), @@ -734,13 +691,10 @@ opts = Options() config_filename = cmd_opts.config opts.load(config_filename) cmd_opts = cmd_args.compatibility_args(opts, cmd_opts) -if cmd_opts.backend == 'diffusers': - log.info('Overriding backend to Diffusers') - opts.data['sd_backend'] = 'Diffusers' -if cmd_opts.backend == 'original': - log.info('Overriding backend to Diffusers') - opts.data['sd_backend'] = 'Original' -backend = Backend.DIFFUSERS if opts.sd_backend == 'Diffusers' else Backend.ORIGINAL +if cmd_opts.backend: + opts.data['sd_backend'] = cmd_opts.backend.lower() +backend = Backend.DIFFUSERS if opts.sd_backend == 'diffusers' else Backend.ORIGINAL +log.info(f'Pipeline: {cmd_opts.backend.lower()}') prompt_styles = modules.styles.StyleDatabase(opts.styles_dir) cmd_opts.disable_extension_access = (cmd_opts.share or cmd_opts.listen or (cmd_opts.server_name or False)) and not cmd_opts.insecure diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index 2fdcb03d9..27170eb40 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -113,33 +113,52 @@ class EmbeddingDatabase: def register_embedding(self, embedding, model): self.word_embeddings[embedding.name] = embedding - ids = model.cond_stage_model.tokenize([embedding.name])[0] - first_id = ids[0] if first_id not in self.ids_lookup: self.ids_lookup[first_id] = [] - self.ids_lookup[first_id] = sorted(self.ids_lookup[first_id] + [(ids, embedding)], key=lambda x: len(x[0]), reverse=True) - return embedding def get_expected_shape(self): if shared.sd_model is None: shared.log.error('Model not loaded') return 0 + if shared.backend == shared.Backend.DIFFUSERS: + return 0 vec = shared.sd_model.cond_stage_model.encode_embedding_init_text(",", 1) return vec.shape[1] + def load_diffusers_embedding(self, filename: str, path: str): + fn, ext = os.path.splitext(filename) + if ext.lower() != ".pt" and ext.lower() != ".safetensors": + return + pipe = shared.sd_model + if filename == "": + pipe.tokenizer = pipe.tokenizer.__class__.from_pretrained(pipe.tokenizer.name_or_path) + pipe.text_encoder.resize_token_embeddings(len(pipe.tokenizer)) + return + name = os.path.basename(fn) + embedding = Embedding(vec=None, name=name) + try: + pipe.load_textual_inversion(path, cache_dir=shared.opts.data["diffusers_dir"], local_files_only=True) + self.word_embeddings[name] = embedding + except Exception: + self.skipped_embeddings[name] = embedding + text_inv_tokens = pipe.tokenizer.added_tokens_encoder.keys() + text_inv_tokens = [t for t in text_inv_tokens if not (len(t.split("_")) > 1 and t.split("_")[-1].isdigit())] + def load_from_file(self, path, filename): name, ext = os.path.splitext(filename) ext = ext.upper() + if shared.backend == shared.Backend.DIFFUSERS: + self.load_diffusers_embedding(filename, path) + return if ext in ['.PNG', '.WEBP', '.JXL', '.AVIF']: _, second_ext = os.path.splitext(name) if second_ext.upper() == '.PREVIEW': return - embed_image = Image.open(path) if hasattr(embed_image, 'text') and 'sd-ti-embedding' in embed_image.text: data = embedding_from_b64(embed_image.text['sd-ti-embedding']) @@ -206,15 +225,12 @@ class EmbeddingDatabase: continue def load_textual_inversion_embeddings(self, force_reload=False): - if shared.backend == shared.Backend.DIFFUSERS: # TODO Diffusers - return if not force_reload: need_reload = False for embdir in self.embedding_dirs.values(): if embdir.has_changed(): need_reload = True break - if not need_reload: return @@ -241,32 +257,25 @@ class EmbeddingDatabase: def find_embedding_at_position(self, tokens, offset): token = tokens[offset] possible_matches = self.ids_lookup.get(token, None) - if possible_matches is None: return None, None - for ids, embedding in possible_matches: if tokens[offset:offset + len(ids)] == ids: return embedding, len(ids) - return None, None def create_embedding(name, num_vectors_per_token, overwrite_old, init_text='*'): cond_model = shared.sd_model.cond_stage_model - with devices.autocast(): cond_model([""]) # will send cond model to GPU if lowvram/medvram is active - #cond_model expects at least some text, so we provide '*' as backup. embedded = cond_model.encode_embedding_init_text(init_text or '*', num_vectors_per_token) vec = torch.zeros((num_vectors_per_token, embedded.shape[1]), device=devices.device) - #Only copy if we provided an init_text, otherwise keep vectors as zeros if init_text: for i in range(num_vectors_per_token): vec[i] = embedded[i * int(embedded.shape[0]) // num_vectors_per_token] - # Remove illegal characters from name. name = "".join( x for x in name if (x.isalnum() or x in "._- ")) fn = os.path.join(shared.opts.embeddings_dir, f"{name}.pt") @@ -299,29 +308,33 @@ def write_loss(log_directory, filename, step, epoch_len, values): **values, }) + def tensorboard_setup(log_directory): os.makedirs(os.path.join(log_directory, "tensorboard"), exist_ok=True) return SummaryWriter( log_dir=os.path.join(log_directory, "tensorboard"), flush_secs=shared.opts.training_tensorboard_flush_every) + def tensorboard_add(tensorboard_writer, loss, global_step, step, learn_rate, epoch_num): tensorboard_add_scaler(tensorboard_writer, "Loss/train", loss, global_step) tensorboard_add_scaler(tensorboard_writer, f"Loss/train/epoch-{epoch_num}", loss, step) tensorboard_add_scaler(tensorboard_writer, "Learn rate/train", learn_rate, global_step) tensorboard_add_scaler(tensorboard_writer, f"Learn rate/train/epoch-{epoch_num}", learn_rate, step) + def tensorboard_add_scaler(tensorboard_writer, tag, value, step): tensorboard_writer.add_scalar(tag=tag, scalar_value=value, global_step=step) + def tensorboard_add_image(tensorboard_writer, tag, pil_image, step): # Convert a pil image to a torch tensor img_tensor = torch.as_tensor(np.array(pil_image, copy=True)) img_tensor = img_tensor.view(pil_image.size[1], pil_image.size[0], len(pil_image.getbands())) img_tensor = img_tensor.permute((2, 0, 1)) - tensorboard_writer.add_image(tag, img_tensor, global_step=step) + def validate_train_inputs(model_name, learn_rate, batch_size, gradient_step, data_root, template_file, template_filename, steps, save_model_every, create_image_every, log_directory, name="embedding"): assert model_name, f"{name} not selected" assert learn_rate, "Learning rate is empty or 0" @@ -383,15 +396,12 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st images_embeds_dir = None hijack = sd_hijack.model_hijack - embedding = hijack.embedding_db.word_embeddings[embedding_name] checkpoint = sd_models.select_checkpoint() - initial_step = embedding.step or 0 if initial_step >= steps: shared.state.textinfo = "Model has already been trained beyond specified max steps" return embedding, filename - scheduler = LearnRateScheduler(learn_rate, steps, initial_step) clip_grad = torch.nn.utils.clip_grad_value_ if clip_grad_mode == "value" else \ torch.nn.utils.clip_grad_norm_ if clip_grad_mode == "norm" else \ diff --git a/modules/ui.py b/modules/ui.py index 2a27df235..42378732b 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -321,7 +321,7 @@ def create_override_settings_dropdown(tabname, row): # pylint: disable=unused-ar return dropdown -def create_ui(): +def create_ui(startup_timer): import modules.img2img # pylint: disable=redefined-outer-name import modules.txt2img # pylint: disable=redefined-outer-name reload_javascript() @@ -334,7 +334,7 @@ def create_ui(): txt_prompt_img = gr.File(label="", elem_id="txt2img_prompt_image", file_count="single", type="binary", visible=False) with FormRow(variant='compact', elem_id="txt2img_extra_networks", visible=False) as extra_networks_ui: from modules import ui_extra_networks - extra_networks_ui = ui_extra_networks.create_ui(extra_networks_ui, extra_networks_button, 'txt2img') + extra_networks_ui = ui_extra_networks.create_ui(extra_networks_ui, extra_networks_button, 'txt2img', skip_indexing=opts.extra_network_skip_indexing) with gr.Row().style(equal_height=False, elem_id="txt2img_interface"): with gr.Column(variant='compact', elem_id="txt2img_settings"): for category in ordered_ui_categories(): @@ -492,9 +492,10 @@ def create_ui(): ui_extra_networks.setup_ui(extra_networks_ui, txt2img_gallery) + startup_timer.record("ui-txt2img") + modules.scripts.scripts_current = modules.scripts.scripts_img2img modules.scripts.scripts_img2img.initialize_scripts(is_img2img=True) - with gr.Blocks(analytics_enabled=False) as img2img_interface: img2img_prompt, img2img_prompt_styles, img2img_negative_prompt, submit, img2img_interrogate, img2img_deepbooru, img2img_prompt_style_apply, img2img_save_style, img2img_paste, extra_networks_button, token_counter, token_button, negative_token_counter, negative_token_button = create_toprow(is_img2img=True) @@ -502,7 +503,7 @@ def create_ui(): with FormRow(variant='compact', elem_id="img2img_extra_networks", visible=False) as extra_networks_ui: from modules import ui_extra_networks - extra_networks_ui_img2img = ui_extra_networks.create_ui(extra_networks_ui, extra_networks_button, 'img2img') + extra_networks_ui_img2img = ui_extra_networks.create_ui(extra_networks_ui, extra_networks_button, 'img2img', skip_indexing=opts.extra_network_skip_indexing) with FormRow().style(equal_height=False, elem_id="img2img_interface"): with gr.Column(variant='compact', elem_id="img2img_settings"): @@ -849,16 +850,21 @@ def create_ui(): paste_button=img2img_paste, tabname="img2img", source_text_component=img2img_prompt, source_image_component=None, )) + startup_timer.record("ui-img2img") + modules.scripts.scripts_current = None with gr.Blocks(analytics_enabled=False) as extras_interface: ui_postprocessing.create_ui() + startup_timer.record("ui-extras") with gr.Blocks(analytics_enabled=False) as train_interface: ui_train.create_ui(txt2img_preview_params = [txt2img_prompt, txt2img_negative_prompt, steps, sampler_index, cfg_scale, seed, width, height]) + startup_timer.record("ui-train") with gr.Blocks(analytics_enabled=False) as models_interface: ui_models.create_ui() + startup_timer.record("ui-models") def create_setting_component(key, is_quicksettings=False): def fun(): @@ -1049,6 +1055,7 @@ def create_ui(): outputs=[dummy_component] ) + startup_timer.record("ui-settings") interfaces = [ (txt2img_interface, "From Text", "txt2img"), @@ -1061,6 +1068,7 @@ def create_ui(): interfaces += [(settings_interface, "Settings", "settings")] extensions_interface = ui_extensions.create_ui() interfaces += [(extensions_interface, "Extensions", "extensions")] + startup_timer.record("ui-extensions") modules.shared.tab_names = [] for _interface, label, _ifid in interfaces: @@ -1136,6 +1144,8 @@ def create_ui(): queue=False, ) + startup_timer.record("ui-defaults") + loadsave.dump_defaults() demo.ui_loadsave = loadsave diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index ba4e7662e..abd0cd80b 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -143,7 +143,10 @@ class ExtraNetworksPage: shared.log.info(f"Extra network created thumbnails: {self.name} {created}") self.missing_thumbs.clear() - def create_html(self, tabname): + def create_html(self, tabname, skip = False): + self_name_id = self.name.replace(" ", "_") + if skip: + return f"
Extra network page not ready
Click refresh to try again
" items_html = '' subdirs = {} allowed_folders = [os.path.abspath(x) for x in self.allowed_directories_for_previews()] @@ -151,7 +154,9 @@ class ExtraNetworksPage: for root, dirs, _files in os.walk(parentdir, followlinks=True): for dirname in dirs: x = os.path.join(root, dirname) - if not os.path.isdir(x): + if shared.opts.diffusers_dir in x: + subdirs[os.path.basename(shared.opts.diffusers_dir)] = 1 + if (not os.path.isdir(x)) or ('models--' in x): continue subdir = os.path.abspath(x)[len(parentdir):].replace("\\", "/") while subdir.startswith("/"): @@ -172,22 +177,15 @@ class ExtraNetworksPage: self.metadata[item["name"]] = item.get("metadata", {}) self.info[item["name"]] = self.find_info(item['filename']) items_html += self.create_html_for_item(item, tabname) - # if items_html == '': - # dirs = "".join([f"
  • {x}
    • " for x in self.allowed_directories_for_previews()]) - # items_html = f'
    No models found: {dirs}
    ' - self_name_id = self.name.replace(" ", "_") if len(subdirs_html) > 0 or len(items_html) > 0: - res = f""" -
    {subdirs_html}
    -
    {items_html}
    - """ + res = f"
    {subdirs_html}
    {items_html}
    " else: return '' threading.Thread(target=self.create_thumb).start() return res except Exception as e: shared.log.error(f'Extra networks page error: {e}') - return '' + return f"
    Extra network error
    {e}
    " def list_items(self): raise NotImplementedError @@ -290,7 +288,7 @@ def sort_extra_pages(pages): return sorted(pages, key=lambda x: tab_scores[x.name]) -def create_ui(container, button, tabname): +def create_ui(container, button, tabname, skip_indexing = False): ui = ExtraNetworksUi() ui.pages = [] ui.stored_extra_pages = sort_extra_pages(extra_pages) @@ -308,7 +306,7 @@ def create_ui(container, button, tabname): ui.description_target_filename = gr.Textbox('Description save filename', elem_id=tabname+"_description_filename", visible=False) for page in ui.stored_extra_pages: - page_html = page.create_html(ui.tabname) + page_html = page.create_html(ui.tabname, skip_indexing) if len(page_html) > 0: with gr.Tab(page.title, id=page.title.lower().replace(" ", "_"), elem_classes="extra-networks-tab"): page_elem = gr.HTML(page_html, elem_id=tabname+page.name+"_extra_page", elem_classes="extra-networks-page") diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index fe65d99df..37bee332b 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -21,10 +21,10 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): "filename": path, "preview": self.find_preview(path), "description": self.find_description(path), - "search_term": self.search_terms_from_path(checkpoint.filename) + " " + (checkpoint.sha256 or ""), + "search_term": f'{self.search_terms_from_path(checkpoint.filename)} {(checkpoint.sha256 or "")} /{checkpoint.type}/', "onclick": '"' + html.escape(f"""return selectCheckpoint({json.dumps(name)})""") + '"', "local_preview": f"{path}.{shared.opts.samples_format}", } def allowed_directories_for_previews(self): - return [v for v in [shared.opts.ckpt_dir, sd_models.model_path] if v is not None] + return [v for v in [shared.opts.ckpt_dir, shared.opts.diffusers_dir, sd_models.model_path] if v is not None] diff --git a/modules/ui_extra_networks_hypernets.py b/modules/ui_extra_networks_hypernets.py index d29863212..01cb22c6c 100644 --- a/modules/ui_extra_networks_hypernets.py +++ b/modules/ui_extra_networks_hypernets.py @@ -19,7 +19,7 @@ class ExtraNetworksPageHypernetworks(ui_extra_networks.ExtraNetworksPage): "preview": self.find_preview(path), "description": self.find_description(path), "search_term": self.search_terms_from_path(path), - "prompt": json.dumps(f""), + "prompt": json.dumps(f""), "local_preview": f"{path}.preview.{shared.opts.samples_format}", } diff --git a/modules/ui_models.py b/modules/ui_models.py index 369a3baec..b7c078a96 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -187,11 +187,11 @@ def create_ui(): def hf_select(evt: gr.SelectData): return data[evt.index[0]][0] - def hf_download_model(hub_id: str): + def hf_download_model(hub_id: str, token): from modules.shared import log, opts from modules.modelloader import download_diffusers_model try: - download_diffusers_model(hub_id, cache_dir=opts.diffusers_dir) + download_diffusers_model(hub_id, cache_dir=opts.diffusers_dir, token=token) except Exception as e: log.error(f"Diffuser model downloaded error: model={hub_id} {e}") return f"Diffuser model downloaded error: model={hub_id} {e}" @@ -200,12 +200,14 @@ def create_ui(): log.info(f"Diffuser model downloaded: model={hub_id}") return f'Diffuser model downloaded: model={hub_id}' - with gr.Row(): - hf_search_text = gr.Textbox('', label = 'Seach models', placeholder='search huggingface models') - - with gr.Row(): - hf_selected = gr.Textbox('', label = 'Select model', placeholder='select model from search results or enter model name manually') - with gr.Row(): + with gr.Column(scale=6): + with gr.Row(): + hf_search_text = gr.Textbox('', label = 'Seach models', placeholder='search huggingface models') + with gr.Row(): + hf_selected = gr.Textbox('', label = 'Select model', placeholder='select model from search results or enter model name manually') + with gr.Row(): + hf_token = gr.Textbox('', label = 'Huggingface token', placeholder='optional access token for private or gated models') + with gr.Column(scale=1): hf_download_model_btn = gr.Button(value="Download model", variant='primary') with gr.Row(): @@ -214,10 +216,9 @@ def create_ui(): hf_search_text.submit(fn=hf_search, inputs=[hf_search_text], outputs=[hf_results]) hf_results.select(hf_select, inputs=None, outputs=[hf_selected]) - hf_download_model_btn.click(fn=hf_download_model, inputs=[hf_selected], outputs=[models_outcome]) + hf_download_model_btn.click(fn=hf_download_model, inputs=[hf_selected, hf_token], outputs=[models_outcome]) # TODO load_diffusers_lora - # TODO load_diffusers_text_inv with gr.Tab(label="CivitAI"): pass diff --git a/webui.py b/webui.py index d95c290b8..b78919c47 100644 --- a/webui.py +++ b/webui.py @@ -229,7 +229,7 @@ def start_ui(): log.debug('Creating UI') modules.script_callbacks.before_ui_callback() startup_timer.record("before-ui") - shared.demo = modules.ui.create_ui() + shared.demo = modules.ui.create_ui(startup_timer) startup_timer.record("ui") if cmd_opts.disable_queue: log.info('Server queues disabled') From bb4a1713fa2d9809ce53eb95edb457961c52d08e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 3 Jul 2023 16:48:25 -0400 Subject: [PATCH 04/42] update extensions --- extensions-builtin/sd-extension-system-info | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index b30e32455..fcdd10c79 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit b30e324552517e68012f2487cf7b0be43616a4cc +Subproject commit fcdd10c7957f85504a4511f2040ec7e01f2054a8 From 35c210655db6ace04938f534853c803341fc16a6 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 3 Jul 2023 17:00:28 -0400 Subject: [PATCH 05/42] linting update --- DIFFUSERS.md | 1 + modules/textual_inversion/image_embedding.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index e43898400..d11ce2c42 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -30,6 +30,7 @@ whats implemented so far? - use huggingface models - extra networks ui - use safetensor models with diffusers backend +- lowvram and medvram equivalents for diffusers - standard workflows: - txt2img, img2img, inpaint, outpaint, process - hires fix, restore faces, etc? diff --git a/modules/textual_inversion/image_embedding.py b/modules/textual_inversion/image_embedding.py index 7ef9b0d68..aca25bc9a 100644 --- a/modules/textual_inversion/image_embedding.py +++ b/modules/textual_inversion/image_embedding.py @@ -42,7 +42,7 @@ def lcg(m=2**32, a=1664525, c=1013904223, seed=0): def xor_block(block): g = lcg() - randblock = np.array([next(g) for _ in range(np.product(block.shape))]).astype(np.uint8).reshape(block.shape) + randblock = np.array([next(g) for _ in range(np.prod(block.shape))]).astype(np.uint8).reshape(block.shape) return np.bitwise_xor(block.astype(np.uint8), randblock & 0x0F) From 8374f08de8b178a2fff2dda1c1ed6da4763157dd Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 3 Jul 2023 17:40:50 -0400 Subject: [PATCH 06/42] fix pipeline --- modules/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shared.py b/modules/shared.py index 5f480f562..e3a0a7704 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -694,7 +694,7 @@ cmd_opts = cmd_args.compatibility_args(opts, cmd_opts) if cmd_opts.backend: opts.data['sd_backend'] = cmd_opts.backend.lower() backend = Backend.DIFFUSERS if opts.sd_backend == 'diffusers' else Backend.ORIGINAL -log.info(f'Pipeline: {cmd_opts.backend.lower()}') +log.info(f'Pipeline: {opts.sd_backend}') prompt_styles = modules.styles.StyleDatabase(opts.styles_dir) cmd_opts.disable_extension_access = (cmd_opts.share or cmd_opts.listen or (cmd_opts.server_name or False)) and not cmd_opts.insecure From 145b990c738ff1d84db006e709043b87543d7e19 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 3 Jul 2023 22:47:29 -0400 Subject: [PATCH 07/42] thumbnail creation exception handling --- DIFFUSERS.md | 2 +- modules/ui_extra_networks.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index d11ce2c42..a0ade1913 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -73,7 +73,7 @@ will need to handle in the code before we get out of alpha ## Issues -- TBD +- seed vs batch size? ## Notes for HF diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index abd0cd80b..cfb4e7525 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -133,12 +133,15 @@ class ExtraNetworksPage: fn = f'{fn}.thumb.jpg' if os.path.exists(fn): continue - created += 1 - img = Image.open(f) - img = img.convert('RGB') - img.thumbnail((512, 512), Image.HAMMING) - img.save(fn) - img.close() + try: + img = Image.open(f) + img = img.convert('RGB') + img.thumbnail((512, 512), Image.HAMMING) + img.save(fn) + img.close() + created += 1 + except Exception as e: + shared.log.error(f'Extra network error creating thumbnail: {f} {e}') if len(self.missing_thumbs) > 0: shared.log.info(f"Extra network created thumbnails: {self.name} {created}") self.missing_thumbs.clear() From b216a35ddd79c02cddfebd80ab999d3a553225bb Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jul 2023 09:28:48 -0400 Subject: [PATCH 08/42] update diffusers and extra networks --- DIFFUSERS.md | 35 +++++++++++++++++++++++++++++------ html/locale_en.json | 6 +++--- javascript/extraNetworks.js | 10 ++++++---- javascript/style.css | 2 +- modules/devices.py | 1 + modules/paths.py | 1 + modules/processing.py | 6 +++--- modules/sd_models.py | 37 +++++++++++++++++++++++++++++++------ modules/shared.py | 9 +++++++++ modules/ui_models.py | 9 ++++----- scripts/xyz_grid.py | 6 +++--- 11 files changed, 91 insertions(+), 31 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index a0ade1913..89b029b2a 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -26,6 +26,7 @@ so diffusers code can be merged into `master` and we can continue with developme whats implemented so far? +- new scheduler: deis - simple model downloader for huggingface models: tabs -> models -> hf hub - use huggingface models - extra networks ui @@ -58,6 +59,7 @@ whats implemented so far? ## Todo - sdxl model +- new schedulers ## Limitations @@ -73,25 +75,46 @@ will need to handle in the code before we get out of alpha ## Issues -- seed vs batch size? +- default model download ckpt vs hfhub? +- new dependency hell (not diffuser related)? +- extra networks ui auto-hide and transitions ## Notes for HF - removed `quicksettings` alternative completely -- added simple model downloader in ui: *tabs -> models -> huggingface* +- added simple model downloader in ui: *tabs -> models -> huggingface* +- attempting to download gated model without access token results in model/refs/commits not found instead of access denied + this is not very user friendly, it should be handled in the code - redone **textual inversion** support, core is now in `modules/textual_inversion/textual_inversion.py:load_diffusers_embedding()` the point is that sdnext pre-loads all compatible embeddings on model load so they are available in prompt context +- redone **lora** support, core is now in `modules/lora_diffusers.py` - added support for diffuser models in **safetensors/ckpt** format - btw, when i use: `diffusers.StableDiffusionPipeline.from_ckpt` +- when i use `diffusers.StableDiffusionPipeline.from_ckpt` first time it downloads something - what is that? > Downloading (…)lve/main/config.json: 4.55k > Downloading pytorch_model.bin: 1.22G - and in general, loading safetensors model is quite slow, is that expected? - for example, 2sec vs 18sec +- loading safetensors model is very slow + for example, 2sec without diffusers and 16sec with diffusers - in `modules/modelloader.py:download_diffusers_model()` i get unknown property for `hf.model_info(hub_id).cardData` can you double-check if this is linter issue or actual problem? -- redone **lora** support, core is now in `modules/lora_diffusers.py` - question on `pipe.load_lora_weights` does it support loading multiple loras? i don't see any notes on that in docs also, lora strength is specified using `cross_attention_kwargs={"scale": x}` during pipeline execution which means if there are multiple loras, they all have the same strength? +- **deepfloyd** failures: + > /home/disty/Apps/automatic/venv/lib/python3.10/site-packages/diffusers/configuration_utils.py:138 in __getattr__ + > AttributeError: 'DDPMScheduler' object has no attribute 'name +- question how do diffusers handle standard 75 token limit for sd? +- diffusers `convert_from_ckpt.py` uses fixed `print` statements so its not possible to control its output to console + it should use `logging` instead. in general, using `print` is bad idea + for example, it very annoyingly logs this every time `StableDiffusionPipeline.from_ckpt` is used: + > global_step key not found in model + > Checkpoint /home/vlado/dev/automatic/models/Stable-diffusion/best/absolutereality_v1.safetensors has both EMA and non-EMA weights. + > In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag. + +## Update + +- sortable models table in downloader ui +- recommended scheduler: `deis` +- `channels_last` and `cudnn_benchmark` now apply to diffusers +- new settings section for diffusers fine-tuning diff --git a/html/locale_en.json b/html/locale_en.json index 3e69cd207..7f828998b 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -524,9 +524,9 @@ ], "scripts": [ {"id":"","label":"Script","localized":"","hint":""}, - {"id":"","label":"Swap X/Y axes","localized":"","hint":""}, - {"id":"","label":"Swap Y/Z axes","localized":"","hint":""}, - {"id":"","label":"Swap X/Z axes","localized":"","hint":""}, + {"id":"","label":"Swap X/Y","localized":"","hint":""}, + {"id":"","label":"Swap Y/Z","localized":"","hint":""}, + {"id":"","label":"Swap X/Z","localized":"","hint":""}, {"id":"","label":"Resize to","localized":"","hint":""}, {"id":"","label":"Resize by","localized":"","hint":""}, {"id":"","label":"Use via API","localized":"","hint":""}, diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 39f75c91c..d4dfc243e 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -9,6 +9,7 @@ function setupExtraNetworksForTab(tabname) { const refresh = gradioApp().getElementById(`${tabname}_extra_refresh`); const description = gradioApp().getElementById(`${tabname}_description`); const close = gradioApp().getElementById(`${tabname}_extra_close`); + const en = gradioApp().getElementById(`${tabname}_extra_networks`); search.classList.add('search'); description.classList.add('description'); tabs.appendChild(refresh); @@ -23,16 +24,15 @@ function setupExtraNetworksForTab(tabname) { elem.style.display = text.indexOf(searchTerm) == -1 ? 'none' : ''; }); }); + intersectionObserver = new IntersectionObserver((entries) => { - // if (entries[0].intersectionRatio <= 0) onHidden(); - const en = gradioApp().getElementById(`${tabname}_extra_networks`); if (entries[0].intersectionRatio > 0) { for (el of Array.from(gradioApp().querySelectorAll('.extra-network-cards'))) { const rect = el.getBoundingClientRect(); - en.style.transition = 'width 0.2s ease'; if (rect.top > 0) { if (!en) return if (window.opts.extra_networks_card_cover == 'cover') { + en.style.transition = ''; en.style.zIndex = 9999; en.style.position = 'absolute'; en.style.right = 'unset'; @@ -40,6 +40,7 @@ function setupExtraNetworksForTab(tabname) { el.style.height = document.body.offsetHeight - el.getBoundingClientRect().top + 'px'; gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' } if (window.opts.extra_networks_card_cover == 'sidebar') { + en.style.transition = 'width 0.2s ease'; en.style.zIndex = 0; en.style.position = 'absolute'; en.style.right = '0'; @@ -47,6 +48,7 @@ function setupExtraNetworksForTab(tabname) { el.style.height = gradioApp().getElementById(`${tabname}_settings`).offsetHeight - 90 + 'px'; gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 100 - 2 - window.opts.extra_networks_sidebar_width + 'vw'; } else { + en.style.transition = ''; en.style.zIndex = 0; en.style.position = 'relative'; en.style.right = 'unset'; @@ -61,7 +63,7 @@ function setupExtraNetworksForTab(tabname) { gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' } }); - intersectionObserver.observe(search); // monitor visibility of + intersectionObserver.observe(en); // monitor visibility of } function setupExtraNetworks() { diff --git a/javascript/style.css b/javascript/style.css index eef8d75ee..2c72ca68e 100644 --- a/javascript/style.css +++ b/javascript/style.css @@ -542,7 +542,7 @@ table.settings-value-table td{ .extra-networks-page { display: flex } .extra-networks .custom-button { min-width: 80px; max-width: 240px; width: 100%; background: none; justify-content: left; text-align: left; padding: 2px 8px 2px 8px; box-shadow: none; } .extra-networks .custom-button:hover { background: var(--button-primary-background-fill) } -.extra-network-cards { display: flex; flex-wrap: wrap; height: 50vh; overflow-y: scroll; overflow-x: hidden; scroll-snap-type: y mandatory; width: -webkit-fill-available; } +.extra-network-cards { display: flex; flex-wrap: wrap; height: 50vh; overflow-y: scroll; overflow-x: hidden; width: -webkit-fill-available; } .extra-network-cards .card { height: fit-content; margin: 0.5em; position: relative; scroll-snap-align: start; scroll-margin-top: 0; } .extra-network-cards .card .overlay { position: absolute; bottom: 0; padding: 0.2em; z-index: 10; width: 100%; background: none; } .extra-network-cards .card:hover .overlay { background: rgba(0, 0, 0, 0.40); } diff --git a/modules/devices.py b/modules/devices.py index 64852d109..eedbb432b 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -122,6 +122,7 @@ def set_cuda_params(): try: torch.backends.cudnn.benchmark = True if shared.opts.cudnn_benchmark: + shared.log.debug('Torch enable cuDNN benchmark') torch.backends.cudnn.benchmark_limit = 0 torch.backends.cudnn.allow_tf32 = shared.opts.cuda_allow_tf32 except Exception: diff --git a/modules/paths.py b/modules/paths.py index deb022c75..966349580 100644 --- a/modules/paths.py +++ b/modules/paths.py @@ -76,6 +76,7 @@ def create_paths(opts): create_path(fix_path('hypernetwork_dir')) create_path(fix_path('ckpt_dir')) create_path(fix_path('vae_dir')) + create_path(fix_path('diffusers_dir')) create_path(fix_path('embeddings_dir')) create_path(fix_path('outdir_samples')) create_path(fix_path('outdir_txt2img_samples')) diff --git a/modules/processing.py b/modules/processing.py index 68c18c977..f50e7482b 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -689,14 +689,14 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: p.scripts.postprocess_batch(p, x_samples_ddim, batch_number=n) elif backend == Backend.DIFFUSERS: - generator = [torch.Generator(device="cpu").manual_seed(s) for s in seeds] + generator_device = 'cpu' if shared.opts.diffusers_generator_device == "cpu" else shared.device + generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds] if shared.sd_model.scheduler.name != p.sampler_name: sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") scheduler = sampler.constructor(shared.sd_model.sd_checkpoint_info.filename) - # TODO(Patrick): For wrapped pipelines this is currently a no-op - shared.sd_model.scheduler = scheduler.sampler + shared.sd_model.scheduler = scheduler.sampler # TODO(Patrick): For wrapped pipelines this is currently a no-op cross_attention_kwargs={} if lora_state['active']: diff --git a/modules/sd_models.py b/modules/sd_models.py index 2181b09d7..186664c95 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -223,7 +223,8 @@ def select_checkpoint(model=True): checkpoint_info = next(iter(checkpoints_list.values())) if model_checkpoint is not None: shared.log.warning(f"Selected checkpoint not found: {model_checkpoint}") - shared.log.warning(f"Loading fallback checkpoint: {checkpoint_info.title}") + # shared.log.warning(f"Loading fallback checkpoint: {checkpoint_info.title}") + shared.opts.data['sd_checkpoint'] = checkpoint_info.title shared.log.debug(f'Select checkpoint: {checkpoint_info.title if checkpoint_info is not None else None}') return checkpoint_info @@ -579,15 +580,39 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No prior = diffusers.DiffusionPipeline.from_pretrained(prior_id, **diffusers_load_config) sd_model = PriorPipeline(prior=prior, main=sd_model) # wrap sd_model - if shared.cmd_opts.medvram: - sd_model.enable_model_cpu_offload() - if shared.cmd_opts.lowvram: - sd_model.enable_sequential_cpu_offload() + if hasattr(sd_model, "enable_sequential_cpu_offload"): + if shared.cmd_opts.lowvram or shared.opts.diffusers_seq_cpu_offload: + sd_model.enable_sequential_cpu_offload() + shared.log.debug('Diffusers: enable sequenctial CPU offload') + if hasattr(sd_model, "enable_model_cpu_offload"): + if shared.cmd_opts.medvram or shared.opts.diffusers_model_cpu_offload: + shared.log.debug('Diffusers: enable model CPU offload') + sd_model.enable_model_cpu_offload() + if hasattr(sd_model, "enable_vae_slicing"): + if shared.opts.diffusers_vae_slicing: + shared.log.debug('Diffusers: enable VAE slicing') + sd_model.enable_vae_slicing() + else: + sd_model.disable_vae_slicing() + if hasattr(sd_model, "enable_vae_tiling"): + if shared.opts.diffusers_vae_tiling: + shared.log.debug('Diffusers: enable VAE tiling') + sd_model.enable_vae_tiling() + else: + sd_model.disable_vae_tiling() + if hasattr(sd_model, "enable_attention_slicing"): + if shared.opts.diffusers_attention_slicing: + shared.log.debug('Diffusers: enable attention slicing') + sd_model.enable_attention_slicing() + else: + sd_model.disable_attention_slicing() if shared.opts.cross_attention_optimization == "xFormers": sd_model.enable_xformers_memory_efficient_attention() + if shared.opts.opt_channelslast: + shared.log.debug('Diffusers: enable channels last') + sd_model.unet.to(memory_format=torch.channels_last) if shared.opts.cuda_compile and torch.cuda.is_available(): sd_model.to(devices.device) - sd_model.unet.to(memory_format=torch.channels_last) import torch._dynamo as dynamo # pylint: disable=unused-import torch._dynamo.config.verbose = shared.opts.cuda_compile_verbose # pylint: disable=protected-access torch._dynamo.config.suppress_errors = shared.opts.cuda_compile_errors # pylint: disable=protected-access diff --git a/modules/shared.py b/modules/shared.py index e3a0a7704..301295723 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -342,6 +342,15 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "disable_gc": OptionInfo(False, "Disable Torch memory garbage collection"), })) +options_templates.update(options_section(('diffusers', "Diffusers Settings"), { + "diffusers_generator_device": OptionInfo("default", "Generator device", gr.Radio, lambda: {"choices": ["default", "cpu"]}), + "diffusers_seq_cpu_offload": OptionInfo(False, "Enable sequential CPU offload"), + "diffusers_model_cpu_offload": OptionInfo(False, "Enable model CPU offload"), + "diffusers_vae_slicing": OptionInfo(False, "Enable VAE slicing"), + "diffusers_vae_tiling": OptionInfo(False, "Enable VAE tiling"), + "diffusers_attention_slicing": OptionInfo(False, "Enable attention slicing"), +})) + options_templates.update(options_section(('system-paths', "System Paths"), { "temp_dir": OptionInfo("", "Directory for temporary images; leave empty for default"), "clean_temp_dir_at_start": OptionInfo(True, "Cleanup non-default temporary directory when starting webui"), diff --git a/modules/ui_models.py b/modules/ui_models.py index b7c078a96..325eca125 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -184,7 +184,7 @@ def create_ui(): data.append([model.modelId, model.pipeline_tag, tags, model.downloads, model.lastModified, f'https://huggingface.co/{model.modelId}']) return data - def hf_select(evt: gr.SelectData): + def hf_select(evt: gr.SelectData, data): return data[evt.index[0]][0] def hf_download_model(hub_id: str, token): @@ -212,13 +212,12 @@ def create_ui(): with gr.Row(): hf_headers = ['Name', 'Pipeline', 'Tags', 'Downloads', 'Updated', 'URL'] - hf_results = gr.DataFrame([], label = 'Search results', show_label = True, interactive = False, wrap = True, overflow_row_behaviour = 'paginate', max_rows = 10, headers = hf_headers, type='array') + hf_types = ['str', 'str', 'str', 'number', 'date', 'markdown'] + hf_results = gr.DataFrame([], label = 'Search results', show_label = True, interactive = False, wrap = True, overflow_row_behaviour = 'paginate', max_rows = 10, headers = hf_headers, datatype = hf_types, type='array') hf_search_text.submit(fn=hf_search, inputs=[hf_search_text], outputs=[hf_results]) - hf_results.select(hf_select, inputs=None, outputs=[hf_selected]) + hf_results.select(fn=hf_select, inputs=[hf_results], outputs=[hf_selected]) hf_download_model_btn.click(fn=hf_download_model, inputs=[hf_selected, hf_token], outputs=[models_outcome]) - # TODO load_diffusers_lora - with gr.Tab(label="CivitAI"): pass diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index 5ecb24134..11f716d00 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -410,9 +410,9 @@ class Script(scripts.Script): with gr.Row(variant="compact", elem_id="axis_options"): margin_size = gr.Slider(label="Grid margins", minimum=0, maximum=500, value=0, step=2, elem_id=self.elem_id("margin_size")) with gr.Row(variant="compact", elem_id="swap_axes"): - swap_xy_axes_button = gr.Button(value="Swap X/Y axes", elem_id="xy_grid_swap_axes_button") - swap_yz_axes_button = gr.Button(value="Swap Y/Z axes", elem_id="yz_grid_swap_axes_button") - swap_xz_axes_button = gr.Button(value="Swap X/Z axes", elem_id="xz_grid_swap_axes_button") + swap_xy_axes_button = gr.Button(value="Swap X/Y", elem_id="xy_grid_swap_axes_button", variant="secondary") + swap_yz_axes_button = gr.Button(value="Swap Y/Z", elem_id="yz_grid_swap_axes_button", variant="secondary") + swap_xz_axes_button = gr.Button(value="Swap X/Z", elem_id="xz_grid_swap_axes_button", variant="secondary") def swap_axes(axis1_type, axis1_values, axis1_values_dropdown, axis2_type, axis2_values, axis2_values_dropdown): return self.current_axis_options[axis2_type].label, axis2_values, axis2_values_dropdown, self.current_axis_options[axis1_type].label, axis1_values, axis1_values_dropdown From 18ef9e6fd717bf7197fc7989f3eee432c092e075 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jul 2023 13:07:05 -0400 Subject: [PATCH 09/42] redo diffusers scheduler --- DIFFUSERS.md | 22 +++++++++++----- html/locale_en.json | 2 +- javascript/style.css | 7 ++--- modules/processing.py | 5 ++-- modules/sd_models.py | 9 ++++--- modules/sd_samplers.py | 15 +++++------ modules/sd_samplers_compvis.py | 1 - modules/sd_samplers_diffusers.py | 44 ++++++++++++++++++++++--------- modules/sd_samplers_diffusors.py | 45 -------------------------------- modules/shared.py | 3 ++- modules/ui_postprocessing.py | 7 ++--- 11 files changed, 69 insertions(+), 91 deletions(-) delete mode 100644 modules/sd_samplers_diffusors.py diff --git a/DIFFUSERS.md b/DIFFUSERS.md index 89b029b2a..6c74ad30b 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -60,6 +60,7 @@ whats implemented so far? - sdxl model - new schedulers +- settings -> schedulers ## Limitations @@ -75,9 +76,7 @@ will need to handle in the code before we get out of alpha ## Issues -- default model download ckpt vs hfhub? - new dependency hell (not diffuser related)? -- extra networks ui auto-hide and transitions ## Notes for HF @@ -90,7 +89,7 @@ will need to handle in the code before we get out of alpha - redone **lora** support, core is now in `modules/lora_diffusers.py` - added support for diffuser models in **safetensors/ckpt** format - when i use `diffusers.StableDiffusionPipeline.from_ckpt` - first time it downloads something - what is that? + first time it downloads something - what is that? (could it be a default safety checker?) > Downloading (…)lve/main/config.json: 4.55k > Downloading pytorch_model.bin: 1.22G - loading safetensors model is very slow @@ -111,10 +110,19 @@ will need to handle in the code before we get out of alpha > global_step key not found in model > Checkpoint /home/vlado/dev/automatic/models/Stable-diffusion/best/absolutereality_v1.safetensors has both EMA and non-EMA weights. > In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag. +- do you have plans to implement [Restart](https://github.com/vladmandic/automatic/issues/1537) sampler in diffusers? +- scheduler config is really difficult to work with as its not possible to see which params each scheduler defines ahead of time and if passing params it doesn't have, it will result in runtime error ## Update -- sortable models table in downloader ui -- recommended scheduler: `deis` -- `channels_last` and `cudnn_benchmark` now apply to diffusers -- new settings section for diffusers fine-tuning +- sortable models table in downloader ui +- system info tab -> benchmark is now working +- recommended scheduler: `deis` +- `channels_last` and `cudnn_benchmark` now apply to diffusers +- new settings section for diffusers fine-tuning +- fixed missed call to `devices.set_cuda_params` +- had to reduce number of supported schedulers by a lot until i add param checking for the rest + issue is that diffusers have completely different params for schedulers than a111, but passing unknown param causes runtime error + previously params were not passed at all, so you couldn't even use anything other than default scheduler (although ui showed you were) +- fixed "it looks like the config file at 'xxx.safetensors' is not a valid JSON file" + this is also related to schedulers as diffusers are trying to read default scheduler config from model itself, but that doesn't exist for safetensors diff --git a/html/locale_en.json b/html/locale_en.json index 7f828998b..3f5617ee4 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -146,7 +146,7 @@ {"id":"","label":"System Paths","localized":"","hint":""}, {"id":"","label":"Image Options","localized":"","hint":""}, {"id":"","label":"Image Processing","localized":"","hint":""}, - {"id":"","label":"Output Paths","localized":"","hint":""}, + {"id":"","label":"Image Paths","localized":"","hint":""}, {"id":"","label":"User Interface","localized":"","hint":""}, {"id":"","label":"Live Previews","localized":"","hint":""}, {"id":"","label":"Sampler Settings","localized":"","hint":""}, diff --git a/javascript/style.css b/javascript/style.css index 2c72ca68e..d079549b8 100644 --- a/javascript/style.css +++ b/javascript/style.css @@ -100,7 +100,8 @@ button.custom-button{ } #txt2img_gallery img, #img2img_gallery img, #extras_gallery img { object-fit: scale-down; width: -webkit-fill-available !important; } -#txt2img_footer, #img2img_footer, #extras_footer{ height: fit-content; display: none; } +#txt2img_footer, #img2img_footer, #extras_footer { height: fit-content; } +#txt2img_footer, #img2img_footer { height: fit-content; display: none; } #txt2img_generate_box, #img2img_generate_box { gap: 0.5em; flex-wrap: wrap-reverse; } #txt2img_actions_column, #img2img_actions_column { gap: 0.5em; } #txt2img_generate_box > button, #img2img_generate_box > button { height: 2.2em; line-height: 0; } @@ -538,11 +539,11 @@ table.settings-value-table td{ .extra-networks .description { margin-top: 8px; } .extra-networks .tab-nav > button { margin-right: 0; height: auto; padding: 2px 4px 2px 4px; } .extra-networks-tab { padding: 0 !important; } -.extra-network-subdirs { background: var(--input-background-fill); } +.extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; max-height: 50vh; } .extra-networks-page { display: flex } .extra-networks .custom-button { min-width: 80px; max-width: 240px; width: 100%; background: none; justify-content: left; text-align: left; padding: 2px 8px 2px 8px; box-shadow: none; } .extra-networks .custom-button:hover { background: var(--button-primary-background-fill) } -.extra-network-cards { display: flex; flex-wrap: wrap; height: 50vh; overflow-y: scroll; overflow-x: hidden; width: -webkit-fill-available; } +.extra-network-cards { display: flex; flex-wrap: wrap; height: 50vh; max-height: 50vh; overflow-y: scroll; overflow-x: hidden; width: -webkit-fill-available; } .extra-network-cards .card { height: fit-content; margin: 0.5em; position: relative; scroll-snap-align: start; scroll-margin-top: 0; } .extra-network-cards .card .overlay { position: absolute; bottom: 0; padding: 0.2em; z-index: 10; width: 100%; background: none; } .extra-network-cards .card:hover .overlay { background: rgba(0, 0, 0, 0.40); } diff --git a/modules/processing.py b/modules/processing.py index f50e7482b..03fd707f1 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -692,11 +692,11 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: generator_device = 'cpu' if shared.opts.diffusers_generator_device == "cpu" else shared.device generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds] if shared.sd_model.scheduler.name != p.sampler_name: + # sd_model.scheduler = diffusers.UniPCMultistepScheduler.from_config(sd_model.scheduler.config) sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") - scheduler = sampler.constructor(shared.sd_model.sd_checkpoint_info.filename) - shared.sd_model.scheduler = scheduler.sampler # TODO(Patrick): For wrapped pipelines this is currently a no-op + shared.sd_model.scheduler = sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op cross_attention_kwargs={} if lora_state['active']: @@ -723,7 +723,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if lora_state['active']: unload_diffusers_lora() - else: raise ValueError(f"Unknown backend {backend}") diff --git a/modules/sd_models.py b/modules/sd_models.py index 186664c95..1a8458bbe 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -541,6 +541,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No "low_cpu_mem_usage": True, "torch_dtype": devices.dtype, "safety_checker": None, + "requires_safety_checker": False, + "load_safety_checker": False, # "use_safetensors": True, # TODO(PVP) - we can't enable this for all checkpoints just yet } @@ -548,6 +550,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No shared.opts.data['sd_model_checkpoint'] = "runwayml/stable-diffusion-v1-5" sd_model = None try: + devices.set_cuda_params() # todo if shared.cmd_opts.ckpt is not None and model_data.initial: # initial load model_name = modelloader.find_diffuser(shared.cmd_opts.ckpt) if model_name is not None: @@ -564,12 +567,12 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) else: diffusers_load_config["local_files_only "] = True - diffusers_load_config["extract_ema"] = True + diffusers_load_config["extract_ema"] = shared.opts.diffusers_extract_ema sd_model = diffusers.StableDiffusionPipeline.from_ckpt(checkpoint_info.path, **diffusers_load_config) if "StableDiffusion" in sd_model.__class__.__name__: - sd_model.scheduler = diffusers.UniPCMultistepScheduler.from_config(sd_model.scheduler.config) - sd_model.scheduler.name = 'UniPC' + from modules.sd_samplers import create_sampler + create_sampler('UniPC', sd_model) elif "Kandinsky" in sd_model.__class__.__name__: sd_model.scheduler.name = 'DDIM' diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index 6033f1bf4..140286064 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -3,14 +3,9 @@ from modules.sd_samplers_common import samples_to_image_grid, sample_to_image # from modules.shared import backend, Backend if backend == Backend.ORIGINAL: - all_samplers = [ - *sd_samplers_kdiffusion.samplers_data_k_diffusion, - *sd_samplers_compvis.samplers_data_compvis, - ] + all_samplers = [*sd_samplers_kdiffusion.samplers_data_k_diffusion, *sd_samplers_compvis.samplers_data_compvis] else: - all_samplers = [ - *sd_samplers_diffusers.samplers_data_diffusers, - ] + all_samplers = [*sd_samplers_diffusers.samplers_data_diffusers] all_samplers_map = {x.name: x for x in all_samplers} samplers = all_samplers samplers_for_img2img = all_samplers @@ -34,10 +29,12 @@ def create_sampler(name, model): sampler = config.constructor(model) sampler.config = config return sampler - else: - sampler = config.constructor(model.sd_checkpoint_info.filename) + elif backend == Backend.DIFFUSERS: + sampler = config.constructor(model) model.scheduler = sampler.sampler return sampler.sampler + else: + return None def set_samplers(): diff --git a/modules/sd_samplers_compvis.py b/modules/sd_samplers_compvis.py index 96bf4d129..562705e1c 100644 --- a/modules/sd_samplers_compvis.py +++ b/modules/sd_samplers_compvis.py @@ -37,7 +37,6 @@ class VanillaStableDiffusionSampler: self.eta = None self.config = None self.last_latent = None - self.conditioning_key = sd_model.model.conditioning_key def number_of_needed_noises(self, p): # pylint: disable=unused-argument diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index ebb988109..bf81ffaeb 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -13,25 +13,43 @@ from diffusers import ( ) from modules import sd_samplers_common +config = { + 'All': { 'num_train_timesteps': 1000, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'linear', 'prediction_type': 'epsilon' }, + 'UniPC': { 'solver_order': 2, 'thresholding': False, 'dynamic_thresholding_ratio': 0.995, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True }, + 'DDIM': { 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'thresholding': False, 'dynamic_thresholding_ratio': 0.995, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'leading', 'rescale_betas_zero_snr': False }, + 'DEIS': { 'solver_order': 2, 'thresholding': False, 'dynamic_thresholding_ratio': 0.995, 'sample_max_value': 1.0, 'algorithm_type': "deis", 'solver_type': "logrho", 'lower_order_final': True }, + 'Euler a': {}, +} + samplers_data_diffusers = [ sd_samplers_common.SamplerData('UniPC', lambda model: DiffusionSampler('UniPC', UniPCMultistepScheduler, model), [], {}), sd_samplers_common.SamplerData('DDIM', lambda model: DiffusionSampler('DDIM', DDIMScheduler, model), [], {}), - sd_samplers_common.SamplerData('DDPM', lambda model: DiffusionSampler('DDPM', DDPMScheduler, model), [], {}), + # sd_samplers_common.SamplerData('DDPM', lambda model: DiffusionSampler('DDPM', DDPMScheduler, model), [], {}), sd_samplers_common.SamplerData('DEIS', lambda model: DiffusionSampler('DEIS', DEISMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 2M', lambda model: DiffusionSampler('DPM++ 2M', DPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 1S', lambda model: DiffusionSampler('DPM++ 1S', DPMSolverSinglestepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 2M SDE', lambda model: DiffusionSampler('DPM++ 2M SDE', DPMSolverMultistepScheduler, model, algorithm_type="sde-dpmsolver++"), [], {}), - sd_samplers_common.SamplerData('DPM++ 2M Karras', lambda model: DiffusionSampler('DPM++ 2M Karras', DPMSolverMultistepScheduler, model, use_karras_sigmas=True), [], {}), - sd_samplers_common.SamplerData('DPM++ 1S Karras', lambda model: DiffusionSampler('DPM++ 1S Karras', DPMSolverSinglestepScheduler, model, use_karras_sigmas=True), [], {}), - sd_samplers_common.SamplerData('DPM++ 2M SDE Karras', lambda model: DiffusionSampler('DPM++ 2M SDE Karras', DPMSolverMultistepScheduler, model, use_karras_sigmas=True, algorithm_type="sde-dpmsolver++"), [], {}), - sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), + # sd_samplers_common.SamplerData('DPM++ 2M', lambda model: DiffusionSampler('DPM++ 2M', DPMSolverMultistepScheduler, model), [], {}), + # sd_samplers_common.SamplerData('DPM++ 1S', lambda model: DiffusionSampler('DPM++ 1S', DPMSolverSinglestepScheduler, model), [], {}), + # sd_samplers_common.SamplerData('DPM++ 2M SDE', lambda model: DiffusionSampler('DPM++ 2M SDE', DPMSolverMultistepScheduler, model, algorithm_type="sde-dpmsolver++"), [], {}), + # sd_samplers_common.SamplerData('DPM++ 2M Karras', lambda model: DiffusionSampler('DPM++ 2M Karras', DPMSolverMultistepScheduler, model, use_karras_sigmas=True), [], {}), + # sd_samplers_common.SamplerData('DPM++ 1S Karras', lambda model: DiffusionSampler('DPM++ 1S Karras', DPMSolverSinglestepScheduler, model, use_karras_sigmas=True), [], {}), + # sd_samplers_common.SamplerData('DPM++ 2M SDE Karras', lambda model: DiffusionSampler('DPM++ 2M SDE Karras', DPMSolverMultistepScheduler, model, use_karras_sigmas=True, algorithm_type="sde-dpmsolver++"), [], {}), + # sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), sd_samplers_common.SamplerData('Euler a', lambda model: DiffusionSampler('Euler a', EulerAncestralDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2++ 2M', lambda model: DiffusionSampler('KDPM2', KDPM2DiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('PNDM', lambda model: DiffusionSampler('PNDM', PNDMScheduler, model), [], {}), + # sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), + # sd_samplers_common.SamplerData('DPM2++ 2M', lambda model: DiffusionSampler('KDPM2', KDPM2DiscreteScheduler, model), [], {}), + # sd_samplers_common.SamplerData('PNDM', lambda model: DiffusionSampler('PNDM', PNDMScheduler, model), [], {}), ] class DiffusionSampler: - def __init__(self, name, constructor, sd_model, **kwargs): - self.sampler = constructor.from_pretrained(sd_model, subfolder="scheduler", **kwargs) + def __init__(self, name, constructor, model, **kwargs): + self.config = config['All'].copy() + for key, value in config.get(name, {}).items(): # diffusers defaults + if key in self.config: + self.config[key] = value + for key, value in model.scheduler.config.items(): # model defaults + if key in self.config: + self.config[key] = value + for key, value in kwargs.items(): # user args + if key in self.config: + self.config[key] = value + self.sampler = constructor(**self.config) self.sampler.name = name diff --git a/modules/sd_samplers_diffusors.py b/modules/sd_samplers_diffusors.py deleted file mode 100644 index 3d102dca4..000000000 --- a/modules/sd_samplers_diffusors.py +++ /dev/null @@ -1,45 +0,0 @@ -from diffusers import ( - DDIMScheduler, - DDPMScheduler, - DEISMultistepScheduler, - DPMSolverMultistepScheduler, - EulerAncestralDiscreteScheduler, - EulerDiscreteScheduler, - HeunDiscreteScheduler, - IPNDMScheduler, - KDPM2AncestralDiscreteScheduler, - PNDMScheduler, - UniPCMultistepScheduler, - # KarrasVeScheduler, - # RePaintScheduler, - # ScoreSdeVeScheduler, - # UnCLIPScheduler, - # VQDiffusionScheduler, -) -from modules import sd_samplers_common - # scheduler = diffusers.UniPCMultistepScheduler.from_pretrained(shared.cmd_opts.ckpt, subfolder="scheduler") - -samplers_data_diffusors = [ - sd_samplers_common.SamplerData('UniPC', lambda model: DiffusionSampler('UniPC', UniPCMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DDIM', lambda model: DiffusionSampler('DDIM', DDIMScheduler, model), [], {}), - sd_samplers_common.SamplerData('DDPMS', lambda model: DiffusionSampler('DDPMS', DDPMScheduler, model), [], {}), - sd_samplers_common.SamplerData('DEIS', lambda model: DiffusionSampler('DEIS', DEISMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPMSolver', lambda model: DiffusionSampler('DPMSolver', DPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('EulerAncestral', lambda model: DiffusionSampler('EulerAncestral', EulerAncestralDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('IPNDM', lambda model: DiffusionSampler('IPNDM', IPNDMScheduler, model), [], {}), - sd_samplers_common.SamplerData('KDPM2Ancestral', lambda model: DiffusionSampler('KDPM2Ancestral', KDPM2AncestralDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('PNDMS', lambda model: DiffusionSampler('PNDMS', PNDMScheduler, model), [], {}), - # sd_samplers_common.SamplerData('KarrasVe', lambda model: DiffusionSampler('KarrasVe', KarrasVeScheduler, model), [], {}), - # sd_samplers_common.SamplerData('RePaint', lambda model: DiffusionSampler('RePaint', RePaintScheduler, model), [], {}), - # sd_samplers_common.SamplerData('ScoreSdeVe', lambda model: DiffusionSampler('ScoreSdeVe', ScoreSdeVeScheduler, model), [], {}), - # sd_samplers_common.SamplerData('UnCLIP', lambda model: DiffusionSampler('UnCLIP', UnCLIPScheduler, model), [], {}), - # sd_samplers_common.SamplerData('VQDiffusion', lambda model: DiffusionSampler('VQDiffusion', VQDiffusionScheduler, model), [], {}), -] - - -class DiffusionSampler: - def __init__(self, name, constructor, sd_model): - self.sampler = constructor.from_pretrained(sd_model, subfolder="scheduler") - self.sampler.name = name diff --git a/modules/shared.py b/modules/shared.py index 301295723..23732b05f 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -343,6 +343,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { })) options_templates.update(options_section(('diffusers', "Diffusers Settings"), { + "diffusers_extract_ema": OptionInfo(True, "Use model EMA weights when possible"), "diffusers_generator_device": OptionInfo("default", "Generator device", gr.Radio, lambda: {"choices": ["default", "cpu"]}), "diffusers_seq_cpu_offload": OptionInfo(False, "Enable sequential CPU offload"), "diffusers_model_cpu_offload": OptionInfo(False, "Enable model CPU offload"), @@ -417,7 +418,7 @@ options_templates.update(options_section(('image-processing', "Image Processing" })) -options_templates.update(options_section(('saving-paths', "Output Paths"), { +options_templates.update(options_section(('saving-paths', "Image Paths"), { "outdir_samples": OptionInfo("", "Output directory for images", component_args=hide_dirs), "outdir_txt2img_samples": OptionInfo("outputs/text", 'Output directory for txt2img images', component_args=hide_dirs), "outdir_img2img_samples": OptionInfo("outputs/image", 'Output directory for img2img images', component_args=hide_dirs), diff --git a/modules/ui_postprocessing.py b/modules/ui_postprocessing.py index b8a82d538..7f5ccc599 100644 --- a/modules/ui_postprocessing.py +++ b/modules/ui_postprocessing.py @@ -9,18 +9,16 @@ from modules.ui_common import infotext_to_html def wrap_pnginfo(image): _, geninfo, info = run_pnginfo(image) - return '', infotext_to_html(geninfo), info, geninfo + return infotext_to_html(geninfo), info, geninfo def submit_click(tab_index, extras_image, image_batch, extras_batch_input_dir, extras_batch_output_dir, show_extras_results, *script_inputs): - result_images, geninfo, js_info = postprocessing.run_postprocessing(tab_index, extras_image, image_batch, extras_batch_input_dir, extras_batch_output_dir, show_extras_results, *script_inputs) return result_images, geninfo, json.dumps(js_info), '' def create_ui(): tab_index = gr.State(value=0) # pylint: disable=abstract-class-instantiated - with gr.Row().style(equal_height=False, variant='compact'): with gr.Column(variant='compact'): with gr.Tabs(elem_id="mode_extras"): @@ -53,11 +51,10 @@ def create_ui(): tab_single.select(fn=lambda: 0, inputs=[], outputs=[tab_index]) tab_batch.select(fn=lambda: 1, inputs=[], outputs=[tab_index]) tab_batch_dir.select(fn=lambda: 2, inputs=[], outputs=[tab_index]) - _dummy = gr.HTML(visible=False) extras_image.change( fn=wrap_gradio_call(wrap_pnginfo), inputs=[extras_image], - outputs=[_dummy, html_info_formatted, exif_info, gen_info], + outputs=[html_info_formatted, exif_info, gen_info], ) submit.click( _js="submit_postprocessing", From 191da73d482d54428d5c3927231d8d0502ce1d73 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jul 2023 14:10:31 -0400 Subject: [PATCH 10/42] diffuser sampler settings --- modules/sd_models.py | 2 +- modules/sd_samplers_diffusers.py | 50 +++++++++++++++++++++----------- modules/shared.py | 50 ++++++++++++++++++++------------ 3 files changed, 65 insertions(+), 37 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index 1a8458bbe..ae65c42fb 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -474,7 +474,7 @@ class SdModelData: elif shared.backend == shared.Backend.DIFFUSERS: load_diffuser() else: - shared.log.error(f"Unknown Stable Diffusion backend: {shared.opts.sd_backend}") + shared.log.error(f"Unknown Stable Diffusion backend: {shared.backend}") self.initial = False except Exception as e: shared.log.error("Failed to load stable diffusion model") diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index bf81ffaeb..fc32513da 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -7,7 +7,7 @@ from diffusers import ( EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, HeunDiscreteScheduler, - KDPM2DiscreteScheduler, + # KDPM2DiscreteScheduler, PNDMScheduler, UniPCMultistepScheduler, ) @@ -15,41 +15,57 @@ from modules import sd_samplers_common config = { 'All': { 'num_train_timesteps': 1000, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'linear', 'prediction_type': 'epsilon' }, - 'UniPC': { 'solver_order': 2, 'thresholding': False, 'dynamic_thresholding_ratio': 0.995, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True }, - 'DDIM': { 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'thresholding': False, 'dynamic_thresholding_ratio': 0.995, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'leading', 'rescale_betas_zero_snr': False }, - 'DEIS': { 'solver_order': 2, 'thresholding': False, 'dynamic_thresholding_ratio': 0.995, 'sample_max_value': 1.0, 'algorithm_type': "deis", 'solver_type': "logrho", 'lower_order_final': True }, + 'UniPC': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True }, + 'DDIM': { 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'thresholding': False, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'leading', 'rescale_betas_zero_snr': False }, + 'DDPM': { 'variance_type': "fixed_small", 'clip_sample': True, 'thresholding': False, 'clip_sample_range': 1.0, 'sample_max_value': 1.0 }, + 'DEIS': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "deis", 'solver_type': "logrho", 'lower_order_final': True }, + 'Euler': { 'interpolation_type': "linear", 'use_karras_sigmas': False }, 'Euler a': {}, + 'Heun': { 'use_karras_sigmas': False }, + 'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0 }, + 'DPM 1S': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False }, + 'DPM 2M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False }, } samplers_data_diffusers = [ sd_samplers_common.SamplerData('UniPC', lambda model: DiffusionSampler('UniPC', UniPCMultistepScheduler, model), [], {}), sd_samplers_common.SamplerData('DDIM', lambda model: DiffusionSampler('DDIM', DDIMScheduler, model), [], {}), - # sd_samplers_common.SamplerData('DDPM', lambda model: DiffusionSampler('DDPM', DDPMScheduler, model), [], {}), + sd_samplers_common.SamplerData('DDPM', lambda model: DiffusionSampler('DDPM', DDPMScheduler, model), [], {}), sd_samplers_common.SamplerData('DEIS', lambda model: DiffusionSampler('DEIS', DEISMultistepScheduler, model), [], {}), - # sd_samplers_common.SamplerData('DPM++ 2M', lambda model: DiffusionSampler('DPM++ 2M', DPMSolverMultistepScheduler, model), [], {}), - # sd_samplers_common.SamplerData('DPM++ 1S', lambda model: DiffusionSampler('DPM++ 1S', DPMSolverSinglestepScheduler, model), [], {}), - # sd_samplers_common.SamplerData('DPM++ 2M SDE', lambda model: DiffusionSampler('DPM++ 2M SDE', DPMSolverMultistepScheduler, model, algorithm_type="sde-dpmsolver++"), [], {}), - # sd_samplers_common.SamplerData('DPM++ 2M Karras', lambda model: DiffusionSampler('DPM++ 2M Karras', DPMSolverMultistepScheduler, model, use_karras_sigmas=True), [], {}), - # sd_samplers_common.SamplerData('DPM++ 1S Karras', lambda model: DiffusionSampler('DPM++ 1S Karras', DPMSolverSinglestepScheduler, model, use_karras_sigmas=True), [], {}), - # sd_samplers_common.SamplerData('DPM++ 2M SDE Karras', lambda model: DiffusionSampler('DPM++ 2M SDE Karras', DPMSolverMultistepScheduler, model, use_karras_sigmas=True, algorithm_type="sde-dpmsolver++"), [], {}), - # sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), + sd_samplers_common.SamplerData('DPM 1S', lambda model: DiffusionSampler('DPM++ 1S', DPMSolverSinglestepScheduler, model), [], {}), + sd_samplers_common.SamplerData('DPM 2M', lambda model: DiffusionSampler('DPM++ 2M', DPMSolverMultistepScheduler, model), [], {}), + sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), sd_samplers_common.SamplerData('Euler a', lambda model: DiffusionSampler('Euler a', EulerAncestralDiscreteScheduler, model), [], {}), - # sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), - # sd_samplers_common.SamplerData('DPM2++ 2M', lambda model: DiffusionSampler('KDPM2', KDPM2DiscreteScheduler, model), [], {}), - # sd_samplers_common.SamplerData('PNDM', lambda model: DiffusionSampler('PNDM', PNDMScheduler, model), [], {}), + sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), + sd_samplers_common.SamplerData('PNDM', lambda model: DiffusionSampler('PNDM', PNDMScheduler, model), [], {}), ] class DiffusionSampler: def __init__(self, name, constructor, model, **kwargs): + from modules.shared import opts, log self.config = config['All'].copy() for key, value in config.get(name, {}).items(): # diffusers defaults - if key in self.config: - self.config[key] = value + self.config[key] = value for key, value in model.scheduler.config.items(): # model defaults if key in self.config: self.config[key] = value for key, value in kwargs.items(): # user args if key in self.config: self.config[key] = value + if opts.schedulers_prediction_type != 'default': + self.config['prediction_type'] = opts.schedulers_prediction_type + if opts.schedulers_beta_schedule != 'default': + self.config['beta_schedule'] = opts.schedulers_beta_schedule + if 'use_karras_sigmas' in self.config: + self.config['use_karras_sigmas'] = opts.schedulers_use_karras + if 'thresholding' in self.config: + self.config['thresholding'] = opts.schedulers_use_thresholding + if 'lower_order_final' in self.config: + self.config['lower_order_final'] = opts.schedulers_use_loworder + if 'solver_order' in self.config: + self.config['solver_order'] = opts.schedulers_solver_order + if name.startswith('DPM'): + self.config['algorithm_type'] = opts.schedulers_dpm_solver self.sampler = constructor(**self.config) self.sampler.name = name + log.debug(f'Diffusers sampler: {name} {self.config}') diff --git a/modules/shared.py b/modules/shared.py index 23732b05f..7ad8db7a8 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -188,6 +188,8 @@ class State: state = State() state.server_start = time.time() +backend = Backend.DIFFUSERS if cmd_opts.backend.lower() == 'diffusers' else Backend.ORIGINAL +log.info(f'Pipeline: {backend}') class OptionInfo: @@ -470,23 +472,37 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"), "show_samplers": OptionInfo(["Euler a", "UniPC", "DDIM", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras", "DEIS"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), "fallback_sampler": OptionInfo("Euler a", "Secondary sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), "force_latent_sampler": OptionInfo("None", "Force latent upscaler sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), - "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems"), - "enable_quantization": OptionInfo(True, "Enable samplers quantization for sharper and cleaner results"), - "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - "eta_ddim": OptionInfo(0.0, "Noise multiplier for DDIM (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - "ddim_discretize": OptionInfo('uniform', "DDIM discretize img2img", gr.Radio, {"choices": ['uniform', 'quad']}), - 's_churn': OptionInfo(0.0, "sigma churn", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - 's_min_uncond': OptionInfo(0, "sigma negative guidance minimum ", gr.Slider, {"minimum": 0.0, "maximum": 4.0, "step": 0.01}), - 's_tmin': OptionInfo(0.0, "sigma tmin", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - 's_noise': OptionInfo(1.0, "sigma noise", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - 'eta_noise_seed_delta': OptionInfo(0, "Noise seed delta (eta)", gr.Number, {"precision": 0}), - 'always_discard_next_to_last_sigma': OptionInfo(False, "Always discard next-to-last sigma"), - 'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}), - 'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}), - 'uni_pc_order': OptionInfo(3, "UniPC order (must be < sampling steps)", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), - 'uni_pc_lower_order_final': OptionInfo(True, "UniPC lower order final"), })) +if backend == Backend.ORIGINAL: + options_templates.update(options_section(('sampler-params', "Sampler Settings"), { + "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems"), + "enable_quantization": OptionInfo(True, "Enable samplers quantization for sharper and cleaner results"), + "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + "eta_ddim": OptionInfo(0.0, "Noise multiplier for DDIM (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + "ddim_discretize": OptionInfo('uniform', "DDIM discretize img2img", gr.Radio, {"choices": ['uniform', 'quad']}), + 's_churn': OptionInfo(0.0, "sigma churn", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + 's_min_uncond': OptionInfo(0, "sigma negative guidance minimum ", gr.Slider, {"minimum": 0.0, "maximum": 4.0, "step": 0.01}), + 's_tmin': OptionInfo(0.0, "sigma tmin", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + 's_noise': OptionInfo(1.0, "sigma noise", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + 'eta_noise_seed_delta': OptionInfo(0, "Noise seed delta (eta)", gr.Number, {"precision": 0}), + 'always_discard_next_to_last_sigma': OptionInfo(False, "Always discard next-to-last sigma"), + 'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}), + 'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}), + 'uni_pc_order': OptionInfo(3, "UniPC order (must be < sampling steps)", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), + 'uni_pc_lower_order_final': OptionInfo(True, "UniPC lower order final"), + })) +elif backend == Backend.DIFFUSERS: + options_templates.update(options_section(('sampler-params', "Sampler Settings"), { + "schedulers_prediction_type": OptionInfo("default", "Samplers override model prediction type", gr.Radio, lambda: {"choices": ['default', 'epsilon', 'sample', 'v-prediction']}), + "schedulers_beta_schedule": OptionInfo("default", "Samplers override beta schedule", gr.Radio, lambda: {"choices": ['default', 'linear', 'scaled_linear', 'squaredcos_cap_v2']}), + "schedulers_solver_order": OptionInfo(2, "Samplers solver order where applicable", gr.Slider, {"minimum": 1, "maximum": 5, "step": 1}), + "schedulers_use_karras": OptionInfo(True, "Samplers should use Karras sigmas where applicable"), + "schedulers_use_loworder": OptionInfo(True, "Samplers should use use lower-order solvers in the final steps where applicable"), + "schedulers_use_thresholding": OptionInfo(False, "Samplers should use dynamic thresholding where applicable"), + "schedulers_dpm_solver": OptionInfo("sde-dpmsolver++", "Samplers DPM solver algorithm", gr.Radio, lambda: {"choices": ['dpmsolver', 'dpmsolver++', 'sde-dpmsolver++']}), + })) + options_templates.update(options_section(('postprocessing', "Postprocessing"), { 'postprocessing_enable_in_main_ui': OptionInfo([], "Enable addtional postprocessing operations", ui_components.DropdownMulti, lambda: {"choices": [x.name for x in shared_items.postprocessing_scripts()]}), 'postprocessing_operation_order': OptionInfo([], "Postprocessing operation order", ui_components.DropdownMulti, lambda: {"choices": [x.name for x in shared_items.postprocessing_scripts()]}), @@ -701,10 +717,6 @@ opts = Options() config_filename = cmd_opts.config opts.load(config_filename) cmd_opts = cmd_args.compatibility_args(opts, cmd_opts) -if cmd_opts.backend: - opts.data['sd_backend'] = cmd_opts.backend.lower() -backend = Backend.DIFFUSERS if opts.sd_backend == 'diffusers' else Backend.ORIGINAL -log.info(f'Pipeline: {opts.sd_backend}') prompt_styles = modules.styles.StyleDatabase(opts.styles_dir) cmd_opts.disable_extension_access = (cmd_opts.share or cmd_opts.listen or (cmd_opts.server_name or False)) and not cmd_opts.insecure From 069c2ddb77f16d9df3a059c5c8303ba4228df15c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jul 2023 15:40:15 -0400 Subject: [PATCH 11/42] fix training --- DIFFUSERS.md | 19 ++++++++++++------- cli/train.py | 11 ++++++++--- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index 6c74ad30b..ef6270a3f 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -59,8 +59,9 @@ whats implemented so far? ## Todo - sdxl model -- new schedulers -- settings -> schedulers +- new schedulers +- settings -> schedulers +- no idea if sd21 works out-of-the-box ## Limitations @@ -84,6 +85,11 @@ will need to handle in the code before we get out of alpha - added simple model downloader in ui: *tabs -> models -> huggingface* - attempting to download gated model without access token results in model/refs/commits not found instead of access denied this is not very user friendly, it should be handled in the code +- redone diffuser sampler support, new code in `modules/sd_samplers_diffusers.py` +- new config section ui settings -> samplers + (its dynamic, it will show standard ui or diffuser specific stuff depending how sdnext is started) +- scheduler config is a bit difficult to work with as its not possible to see which params each scheduler defines ahead of time + and if passing params it doesn't have, it will result in runtime error - redone **textual inversion** support, core is now in `modules/textual_inversion/textual_inversion.py:load_diffusers_embedding()` the point is that sdnext pre-loads all compatible embeddings on model load so they are available in prompt context - redone **lora** support, core is now in `modules/lora_diffusers.py` @@ -100,6 +106,8 @@ will need to handle in the code before we get out of alpha does it support loading multiple loras? i don't see any notes on that in docs also, lora strength is specified using `cross_attention_kwargs={"scale": x}` during pipeline execution which means if there are multiple loras, they all have the same strength? +- any plans to support more complex loras? from limited testing + it seems only basic loras are working while lycoris/locon are not - **deepfloyd** failures: > /home/disty/Apps/automatic/venv/lib/python3.10/site-packages/diffusers/configuration_utils.py:138 in __getattr__ > AttributeError: 'DDPMScheduler' object has no attribute 'name @@ -111,7 +119,6 @@ will need to handle in the code before we get out of alpha > Checkpoint /home/vlado/dev/automatic/models/Stable-diffusion/best/absolutereality_v1.safetensors has both EMA and non-EMA weights. > In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag. - do you have plans to implement [Restart](https://github.com/vladmandic/automatic/issues/1537) sampler in diffusers? -- scheduler config is really difficult to work with as its not possible to see which params each scheduler defines ahead of time and if passing params it doesn't have, it will result in runtime error ## Update @@ -121,8 +128,6 @@ will need to handle in the code before we get out of alpha - `channels_last` and `cudnn_benchmark` now apply to diffusers - new settings section for diffusers fine-tuning - fixed missed call to `devices.set_cuda_params` -- had to reduce number of supported schedulers by a lot until i add param checking for the rest - issue is that diffusers have completely different params for schedulers than a111, but passing unknown param causes runtime error - previously params were not passed at all, so you couldn't even use anything other than default scheduler (although ui showed you were) +- redid samplers - fixed "it looks like the config file at 'xxx.safetensors' is not a valid JSON file" - this is also related to schedulers as diffusers are trying to read default scheduler config from model itself, but that doesn't exist for safetensors +- ui settings -> samplers is now dynamic depending if backend is original or diffusers diff --git a/cli/train.py b/cli/train.py index cf27fbcc8..97268dd54 100755 --- a/cli/train.py +++ b/cli/train.py @@ -265,6 +265,7 @@ def prepare_options(): options.lora.in_json = None if args.type == 'dreambooth': log.info('train using dreambooth style training') + options.lora.vae_batch_size = args.batch options.lora.in_json = None if args.type == 'lora': log.info('train using lora style training') @@ -378,14 +379,18 @@ def check_versions(): log.info('experimental mode enabled') return log.info('checking accelerate') + error = False import accelerate if accelerate.__version__ != '0.19.0': - log.error(f'invalid accelerate version: required=0.19.0 found={accelerate.__version__}') - exit(1) + log.error(f'invalid accelerate version: accelerate=0.19.0 found={accelerate.__version__}') + error = True log.info('checking diffusers') import diffusers if diffusers.__version__ != '0.10.2': - log.error(f'invalid diffusers version: required=0.10.2 found={diffusers.__version__}') + log.error(f'invalid diffusers version: diffusers=0.10.2 found={diffusers.__version__}') + error = True + if error: + log.info('> pip install accelerate==0.19.0 diffusers==0.10.2') exit(1) From a0e1c898b75c6f6382c30be1988239b45e60d931 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jul 2023 15:50:44 -0400 Subject: [PATCH 12/42] fix argparse --- modules/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shared.py b/modules/shared.py index 7ad8db7a8..ba638ebd3 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -188,7 +188,7 @@ class State: state = State() state.server_start = time.time() -backend = Backend.DIFFUSERS if cmd_opts.backend.lower() == 'diffusers' else Backend.ORIGINAL +backend = Backend.DIFFUSERS if (cmd_opts.backend is not None) and (cmd_opts.backend.lower() == 'diffusers') else Backend.ORIGINAL log.info(f'Pipeline: {backend}') From 2524b6659c9ee26811f561cfaea610439113a837 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jul 2023 16:04:22 -0400 Subject: [PATCH 13/42] double package install pass --- installer.py | 4 ++-- launch.py | 1 + modules/processing.py | 3 +-- modules/sd_models.py | 3 +-- modules/shared.py | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/installer.py b/installer.py index 4e9a3b805..220d20e93 100644 --- a/installer.py +++ b/installer.py @@ -414,7 +414,7 @@ def install_repositories(): pr.enable() def d(name): return os.path.join(os.path.dirname(__file__), 'repositories', name) - log.info('Installing repositories') + log.info('Verifying repositories') os.makedirs(os.path.join(os.path.dirname(__file__), 'repositories'), exist_ok=True) stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") # stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf") @@ -526,7 +526,7 @@ def install_submodules(): if args.profile: pr = cProfile.Profile() pr.enable() - log.info('Installing submodules') + log.info('Verifying submodules') txt = git('submodule') log.debug(f'Submodules list: {txt}') if 'no submodule mapping found' in txt: diff --git a/launch.py b/launch.py index aea1b404b..cac19f277 100644 --- a/launch.py +++ b/launch.py @@ -174,6 +174,7 @@ if __name__ == "__main__": installer.install_repositories() installer.install_submodules() installer.install_extensions() + installer.install_packages() # redo packages since extensions may change them installer.update_wiki() if installer.errors == 0: installer.log.debug(f'Setup complete without errors: {round(time.time())}') diff --git a/modules/processing.py b/modules/processing.py index 03fd707f1..f0e55acc2 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -691,8 +691,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: elif backend == Backend.DIFFUSERS: generator_device = 'cpu' if shared.opts.diffusers_generator_device == "cpu" else shared.device generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds] - if shared.sd_model.scheduler.name != p.sampler_name: - # sd_model.scheduler = diffusers.UniPCMultistepScheduler.from_config(sd_model.scheduler.config) + if (not hasattr(shared.sd_model.scheduler, 'name')) or (shared.sd_model.scheduler.name != p.sampler_name): sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") diff --git a/modules/sd_models.py b/modules/sd_models.py index ae65c42fb..66df8a0ac 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -571,8 +571,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model = diffusers.StableDiffusionPipeline.from_ckpt(checkpoint_info.path, **diffusers_load_config) if "StableDiffusion" in sd_model.__class__.__name__: - from modules.sd_samplers import create_sampler - create_sampler('UniPC', sd_model) + pass # scheduler is created on first use elif "Kandinsky" in sd_model.__class__.__name__: sd_model.scheduler.name = 'DDIM' diff --git a/modules/shared.py b/modules/shared.py index ba638ebd3..c9aae6b90 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -469,7 +469,7 @@ options_templates.update(options_section(('live-preview', "Live Previews"), { options_templates.update(options_section(('sampler-params', "Sampler Settings"), { - "show_samplers": OptionInfo(["Euler a", "UniPC", "DDIM", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras", "DEIS"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), + "show_samplers": OptionInfo(["Euler a", "UniPC", "DEIS", "DDIM", "DPM 1S", "DPM 2M", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), "fallback_sampler": OptionInfo("Euler a", "Secondary sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), "force_latent_sampler": OptionInfo("None", "Force latent upscaler sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), })) From 3eff89f1aba6ce146ea9cb518bc8e682fef1567e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jul 2023 20:30:37 -0400 Subject: [PATCH 14/42] fix extra networks folder view width --- javascript/style.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/javascript/style.css b/javascript/style.css index d079549b8..6340485f2 100644 --- a/javascript/style.css +++ b/javascript/style.css @@ -539,9 +539,9 @@ table.settings-value-table td{ .extra-networks .description { margin-top: 8px; } .extra-networks .tab-nav > button { margin-right: 0; height: auto; padding: 2px 4px 2px 4px; } .extra-networks-tab { padding: 0 !important; } -.extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; max-height: 50vh; } +.extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; max-height: 50vh; min-width: 80px; max-width: 120px; } .extra-networks-page { display: flex } -.extra-networks .custom-button { min-width: 80px; max-width: 240px; width: 100%; background: none; justify-content: left; text-align: left; padding: 2px 8px 2px 8px; box-shadow: none; } +.extra-networks .custom-button { min-width: 80px; max-width: 120px; width: 100%; background: none; justify-content: left; text-align: left; padding: 2px 8px 2px 8px; box-shadow: none; line-break: anywhere; } .extra-networks .custom-button:hover { background: var(--button-primary-background-fill) } .extra-network-cards { display: flex; flex-wrap: wrap; height: 50vh; max-height: 50vh; overflow-y: scroll; overflow-x: hidden; width: -webkit-fill-available; } .extra-network-cards .card { height: fit-content; margin: 0.5em; position: relative; scroll-snap-align: start; scroll-margin-top: 0; } From 65f036e42a545f2fbcd8107404915a968834e857 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 5 Jul 2023 09:47:07 -0400 Subject: [PATCH 15/42] add pan and zoom controls to image viewer --- CHANGELOG.md | 6 +- DIFFUSERS.md | 1 + README.md | 13 + TODO.md | 6 +- javascript/imageViewer.js | 19 +- javascript/panZoom.js | 1768 +++++++++++++++++++++++++++++++++++++ 6 files changed, 1797 insertions(+), 16 deletions(-) create mode 100644 javascript/panZoom.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 91ef21dc7..4c4447625 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,13 @@ # Change Log for SD.Next -## Update for ... +## Update for 07/07/2023 +- add pan & zoom controls (touch and mouse) to image viewer (lightbox) - add settings -> extra networks -> do not automatically build extra network pages speeds up app start if you have a lot of extra networks and you want to build them manually when needed +- extra network ui tweaks +- merge experimental diffusers support + this will be covered in details in separate post ## Update for 07/01/2023 diff --git a/DIFFUSERS.md b/DIFFUSERS.md index ef6270a3f..ac9dc998e 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -119,6 +119,7 @@ will need to handle in the code before we get out of alpha > Checkpoint /home/vlado/dev/automatic/models/Stable-diffusion/best/absolutereality_v1.safetensors has both EMA and non-EMA weights. > In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag. - do you have plans to implement [Restart](https://github.com/vladmandic/automatic/issues/1537) sampler in diffusers? +- `torch.nonzero()` performance issue ## Update diff --git a/README.md b/README.md index eb9bcbac2..4e8610d4d 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,19 @@ Below is partial list of all available parameters, run `webui --help` for the fu ## Notes +### **Extensions** + +SD.Next comes with several extensions pre-installed: + +- [Dyanmic Thresholding](https://github.com/mcmonkeyprojects/sd-dynamic-thresholding) +- [ControlNet](https://github.com/Mikubill/sd-webui-controlnet) +- [Agent Scheduler](https://github.com/ArtVentureX/sd-webui-agent-scheduler) +- [Multi-Diffusion Tiled Diffusion and VAE](https://github.com/pkuliyi2015/multidiffusion-upscaler-for-automatic1111) +- [LyCORIS](https://github.com/KohakuBlueleaf/a1111-sd-webui-lycoris) +- [Image Browser](https://github.com/AlUlkesh/stable-diffusion-webui-images-browser) +- [CLiP Interrogator](https://github.com/pharmapsychotic/clip-interrogator-ext) +- [Rembg Background Removal](https://github.com/AUTOMATIC1111/stable-diffusion-webui-rembg) + ### **Collab** - To avoid having this repo rely just on me, I'd love to have additional maintainers with full admin rights. If you're interested, ping me! diff --git a/TODO.md b/TODO.md index ef0e08985..60ced2f41 100644 --- a/TODO.md +++ b/TODO.md @@ -26,7 +26,8 @@ Stuff to be investigated... Pick & merge PRs from main repo... -- list: +- up-to-date with: df004be +- current todo list: ## Integration @@ -48,7 +49,6 @@ Tech that can be integrated as part of the core workflow... - Bunch of stuff: - -- shared.info - docker - port `p.all_hr_prompts` - test `lyco_patch_lora` @@ -57,6 +57,6 @@ Tech that can be integrated as part of the core workflow... - image watermark - image `imagehash` phash and hdash - custom exif tags -- replace lightbox with iv-viewer - git-rebasin - additional upscalers +- new image browser diff --git a/javascript/imageViewer.js b/javascript/imageViewer.js index c4cac2f90..4a5302fcc 100644 --- a/javascript/imageViewer.js +++ b/javascript/imageViewer.js @@ -149,9 +149,8 @@ onAfterUiUpdate(() => { }); document.addEventListener('DOMContentLoaded', () => { - // const modalFragment = document.createDocumentFragment(); const modal = document.createElement('div'); - modal.onclick = closeModal; + // modal.onclick = closeModal; modal.id = 'lightboxModal'; modal.tabIndex = 0; modal.addEventListener('keydown', modalKeyHandler, true); @@ -182,21 +181,17 @@ document.addEventListener('DOMContentLoaded', () => { modalSave.title = 'Save Image(s)'; modalControls.appendChild(modalSave); - /* - const modalClose = document.createElement('span'); - modalClose.className = 'modalClose cursor'; - modalClose.innerHTML = '×'; - modalClose.onclick = closeModal; - modalClose.title = 'Close image viewer'; - modalControls.appendChild(modalClose); - */ - const modalImage = document.createElement('img'); modalImage.id = 'modalImage'; - modalImage.onclick = closeModal; modalImage.tabIndex = 0; modalImage.addEventListener('keydown', modalKeyHandler, true); modal.appendChild(modalImage); + modalImage.onload = () => panzoom(modalImage, { zoomSpeed: 0.025, minZoom: 0.25, maxZoom: 4.0 }); + let drag = false; + modalImage.addEventListener('mousedown', () => drag = false); + modalImage.addEventListener('mousemove', () => drag = true); + modalImage.addEventListener('mouseup', () => { if (!drag) closeModal(); }); + // modalImage.onclick = closeModal; const modalPrev = document.createElement('a'); modalPrev.className = 'modalPrev'; diff --git a/javascript/panZoom.js b/javascript/panZoom.js new file mode 100644 index 000000000..fec42c597 --- /dev/null +++ b/javascript/panZoom.js @@ -0,0 +1,1768 @@ +(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.panzoom = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i 0) { + transform.x += diff; + adjusted = true; + } + // check the other side: + diff = boundingBox.right - clientRect.left; + if (diff < 0) { + transform.x += diff; + adjusted = true; + } + + // y axis: + diff = boundingBox.top - clientRect.bottom; + if (diff > 0) { + // we adjust transform, so that it matches exactly our bounding box: + // transform.y = boundingBox.top - (boundingBox.height + boundingBox.y) * transform.scale => + // transform.y = boundingBox.top - (clientRect.bottom - transform.y) => + // transform.y = diff + transform.y => + transform.y += diff; + adjusted = true; + } + + diff = boundingBox.bottom - clientRect.top; + if (diff < 0) { + transform.y += diff; + adjusted = true; + } + return adjusted; + } + + /** + * Returns bounding box that should be used to restrict scene movement. + */ + function getBoundingBox() { + if (!bounds) return; // client does not want to restrict movement + + if (typeof bounds === 'boolean') { + // for boolean type we use parent container bounds + var ownerRect = owner.getBoundingClientRect(); + var sceneWidth = ownerRect.width; + var sceneHeight = ownerRect.height; + + return { + left: sceneWidth * boundsPadding, + top: sceneHeight * boundsPadding, + right: sceneWidth * (1 - boundsPadding), + bottom: sceneHeight * (1 - boundsPadding) + }; + } + + return bounds; + } + + function getClientRect() { + var bbox = panController.getBBox(); + var leftTop = client(bbox.left, bbox.top); + + return { + left: leftTop.x, + top: leftTop.y, + right: bbox.width * transform.scale + leftTop.x, + bottom: bbox.height * transform.scale + leftTop.y + }; + } + + function client(x, y) { + return { + x: x * transform.scale + transform.x, + y: y * transform.scale + transform.y + }; + } + + function makeDirty() { + isDirty = true; + frameAnimation = window.requestAnimationFrame(frame); + } + + function zoomByRatio(clientX, clientY, ratio) { + if (isNaN(clientX) || isNaN(clientY) || isNaN(ratio)) { + throw new Error('zoom requires valid numbers'); + } + + var newScale = transform.scale * ratio; + + if (newScale < minZoom) { + if (transform.scale === minZoom) return; + + ratio = minZoom / transform.scale; + } + if (newScale > maxZoom) { + if (transform.scale === maxZoom) return; + + ratio = maxZoom / transform.scale; + } + + var size = transformToScreen(clientX, clientY); + + transform.x = size.x - ratio * (size.x - transform.x); + transform.y = size.y - ratio * (size.y - transform.y); + + // TODO: https://github.com/anvaka/panzoom/issues/112 + if (bounds && boundsPadding === 1 && minZoom === 1) { + transform.scale *= ratio; + keepTransformInsideBounds(); + } else { + var transformAdjusted = keepTransformInsideBounds(); + if (!transformAdjusted) transform.scale *= ratio; + } + + triggerEvent('zoom'); + + makeDirty(); + } + + function zoomAbs(clientX, clientY, zoomLevel) { + var ratio = zoomLevel / transform.scale; + zoomByRatio(clientX, clientY, ratio); + } + + function centerOn(ui) { + var parent = ui.ownerSVGElement; + if (!parent) + throw new Error('ui element is required to be within the scene'); + + // TODO: should i use controller's screen CTM? + var clientRect = ui.getBoundingClientRect(); + var cx = clientRect.left + clientRect.width / 2; + var cy = clientRect.top + clientRect.height / 2; + + var container = parent.getBoundingClientRect(); + var dx = container.width / 2 - cx; + var dy = container.height / 2 - cy; + + internalMoveBy(dx, dy, true); + } + + function smoothMoveTo(x, y){ + internalMoveBy(x - transform.x, y - transform.y, true) + } + + function internalMoveBy(dx, dy, smooth) { + if (!smooth) { + return moveBy(dx, dy); + } + + if (moveByAnimation) moveByAnimation.cancel(); + + var from = { x: 0, y: 0 }; + var to = { x: dx, y: dy }; + var lastX = 0; + var lastY = 0; + + moveByAnimation = animate(from, to, { + step: function (v) { + moveBy(v.x - lastX, v.y - lastY); + + lastX = v.x; + lastY = v.y; + } + }); + } + + function scroll(x, y) { + cancelZoomAnimation(); + moveTo(x, y); + } + + function dispose() { + releaseEvents(); + } + + function listenForEvents() { + owner.addEventListener('mousedown', onMouseDown, { passive: false }); + owner.addEventListener('dblclick', onDoubleClick, { passive: false }); + owner.addEventListener('touchstart', onTouch, { passive: false }); + owner.addEventListener('keydown', onKeyDown, { passive: false }); + + // Need to listen on the owner container, so that we are not limited + // by the size of the scrollable domElement + wheel.addWheelListener(owner, onMouseWheel, { passive: false }); + + makeDirty(); + } + + function releaseEvents() { + wheel.removeWheelListener(owner, onMouseWheel); + owner.removeEventListener('mousedown', onMouseDown); + owner.removeEventListener('keydown', onKeyDown); + owner.removeEventListener('dblclick', onDoubleClick); + owner.removeEventListener('touchstart', onTouch); + + if (frameAnimation) { + window.cancelAnimationFrame(frameAnimation); + frameAnimation = 0; + } + + smoothScroll.cancel(); + + releaseDocumentMouse(); + releaseTouches(); + textSelection.release(); + + triggerPanEnd(); + } + + function frame() { + if (isDirty) applyTransform(); + } + + function applyTransform() { + isDirty = false; + + // TODO: Should I allow to cancel this? + panController.applyTransform(transform); + + triggerEvent('transform'); + frameAnimation = 0; + } + + function onKeyDown(e) { + var x = 0, + y = 0, + z = 0; + if (e.keyCode === 38) { + y = 1; // up + } else if (e.keyCode === 40) { + y = -1; // down + } else if (e.keyCode === 37) { + x = 1; // left + } else if (e.keyCode === 39) { + x = -1; // right + } else if (e.keyCode === 189 || e.keyCode === 109) { + // DASH or SUBTRACT + z = 1; // `-` - zoom out + } else if (e.keyCode === 187 || e.keyCode === 107) { + // EQUAL SIGN or ADD + z = -1; // `=` - zoom in (equal sign on US layout is under `+`) + } + + if (filterKey(e, x, y, z)) { + // They don't want us to handle the key: https://github.com/anvaka/panzoom/issues/45 + return; + } + + if (x || y) { + e.preventDefault(); + e.stopPropagation(); + + var clientRect = owner.getBoundingClientRect(); + // movement speed should be the same in both X and Y direction: + var offset = Math.min(clientRect.width, clientRect.height); + var moveSpeedRatio = 0.05; + var dx = offset * moveSpeedRatio * x; + var dy = offset * moveSpeedRatio * y; + + // TODO: currently we do not animate this. It could be better to have animation + internalMoveBy(dx, dy); + } + + if (z) { + var scaleMultiplier = getScaleMultiplier(z * 100); + var offset = transformOrigin ? getTransformOriginOffset() : midPoint(); + publicZoomTo(offset.x, offset.y, scaleMultiplier); + } + } + + function midPoint() { + var ownerRect = owner.getBoundingClientRect(); + return { + x: ownerRect.width / 2, + y: ownerRect.height / 2 + }; + } + + function onTouch(e) { + // let the override the touch behavior + beforeTouch(e); + + if (e.touches.length === 1) { + return handleSingleFingerTouch(e, e.touches[0]); + } else if (e.touches.length === 2) { + // handleTouchMove() will care about pinch zoom. + pinchZoomLength = getPinchZoomLength(e.touches[0], e.touches[1]); + multiTouch = true; + startTouchListenerIfNeeded(); + } + } + + function beforeTouch(e) { + // TODO: Need to unify this filtering names. E.g. use `beforeTouch` + if (options.onTouch && !options.onTouch(e)) { + // if they return `false` from onTouch, we don't want to stop + // events propagation. Fixes https://github.com/anvaka/panzoom/issues/12 + return; + } + + e.stopPropagation(); + e.preventDefault(); + } + + function beforeDoubleClick(e) { + // TODO: Need to unify this filtering names. E.g. use `beforeDoubleClick`` + if (options.onDoubleClick && !options.onDoubleClick(e)) { + // if they return `false` from onTouch, we don't want to stop + // events propagation. Fixes https://github.com/anvaka/panzoom/issues/46 + return; + } + + e.preventDefault(); + e.stopPropagation(); + } + + function handleSingleFingerTouch(e) { + var touch = e.touches[0]; + var offset = getOffsetXY(touch); + lastSingleFingerOffset = offset; + var point = transformToScreen(offset.x, offset.y); + mouseX = point.x; + mouseY = point.y; + + smoothScroll.cancel(); + startTouchListenerIfNeeded(); + } + + function startTouchListenerIfNeeded() { + if (touchInProgress) { + // no need to do anything, as we already listen to events; + return; + } + + touchInProgress = true; + document.addEventListener('touchmove', handleTouchMove); + document.addEventListener('touchend', handleTouchEnd); + document.addEventListener('touchcancel', handleTouchEnd); + } + + function handleTouchMove(e) { + if (e.touches.length === 1) { + e.stopPropagation(); + var touch = e.touches[0]; + + var offset = getOffsetXY(touch); + var point = transformToScreen(offset.x, offset.y); + + var dx = point.x - mouseX; + var dy = point.y - mouseY; + + if (dx !== 0 && dy !== 0) { + triggerPanStart(); + } + mouseX = point.x; + mouseY = point.y; + internalMoveBy(dx, dy); + } else if (e.touches.length === 2) { + // it's a zoom, let's find direction + multiTouch = true; + var t1 = e.touches[0]; + var t2 = e.touches[1]; + var currentPinchLength = getPinchZoomLength(t1, t2); + + // since the zoom speed is always based on distance from 1, we need to apply + // pinch speed only on that distance from 1: + var scaleMultiplier = + 1 + (currentPinchLength / pinchZoomLength - 1) * pinchSpeed; + + var firstTouchPoint = getOffsetXY(t1); + var secondTouchPoint = getOffsetXY(t2); + mouseX = (firstTouchPoint.x + secondTouchPoint.x) / 2; + mouseY = (firstTouchPoint.y + secondTouchPoint.y) / 2; + if (transformOrigin) { + var offset = getTransformOriginOffset(); + mouseX = offset.x; + mouseY = offset.y; + } + + publicZoomTo(mouseX, mouseY, scaleMultiplier); + + pinchZoomLength = currentPinchLength; + e.stopPropagation(); + e.preventDefault(); + } + } + + function handleTouchEnd(e) { + if (e.touches.length > 0) { + var offset = getOffsetXY(e.touches[0]); + var point = transformToScreen(offset.x, offset.y); + mouseX = point.x; + mouseY = point.y; + } else { + var now = new Date(); + if (now - lastTouchEndTime < doubleTapSpeedInMS) { + if (transformOrigin) { + var offset = getTransformOriginOffset(); + smoothZoom(offset.x, offset.y, zoomDoubleClickSpeed); + } else { + // We want untransformed x/y here. + smoothZoom(lastSingleFingerOffset.x, lastSingleFingerOffset.y, zoomDoubleClickSpeed); + } + } + + lastTouchEndTime = now; + + triggerPanEnd(); + releaseTouches(); + } + } + + function getPinchZoomLength(finger1, finger2) { + var dx = finger1.clientX - finger2.clientX; + var dy = finger1.clientY - finger2.clientY; + return Math.sqrt(dx * dx + dy * dy); + } + + function onDoubleClick(e) { + beforeDoubleClick(e); + var offset = getOffsetXY(e); + if (transformOrigin) { + // TODO: looks like this is duplicated in the file. + // Need to refactor + offset = getTransformOriginOffset(); + } + smoothZoom(offset.x, offset.y, zoomDoubleClickSpeed); + } + + function onMouseDown(e) { + // if client does not want to handle this event - just ignore the call + if (beforeMouseDown(e)) return; + + if (touchInProgress) { + // modern browsers will fire mousedown for touch events too + // we do not want this: touch is handled separately. + e.stopPropagation(); + return false; + } + // for IE, left click == 1 + // for Firefox, left click == 0 + var isLeftButton = + (e.button === 1 && window.event !== null) || e.button === 0; + if (!isLeftButton) return; + + smoothScroll.cancel(); + + var offset = getOffsetXY(e); + var point = transformToScreen(offset.x, offset.y); + mouseX = point.x; + mouseY = point.y; + + // We need to listen on document itself, since mouse can go outside of the + // window, and we will loose it + document.addEventListener('mousemove', onMouseMove); + document.addEventListener('mouseup', onMouseUp); + textSelection.capture(e.target || e.srcElement); + + return false; + } + + function onMouseMove(e) { + // no need to worry about mouse events when touch is happening + if (touchInProgress) return; + + triggerPanStart(); + + var offset = getOffsetXY(e); + var point = transformToScreen(offset.x, offset.y); + var dx = point.x - mouseX; + var dy = point.y - mouseY; + + mouseX = point.x; + mouseY = point.y; + + internalMoveBy(dx, dy); + } + + function onMouseUp() { + textSelection.release(); + triggerPanEnd(); + releaseDocumentMouse(); + } + + function releaseDocumentMouse() { + document.removeEventListener('mousemove', onMouseMove); + document.removeEventListener('mouseup', onMouseUp); + panstartFired = false; + } + + function releaseTouches() { + document.removeEventListener('touchmove', handleTouchMove); + document.removeEventListener('touchend', handleTouchEnd); + document.removeEventListener('touchcancel', handleTouchEnd); + panstartFired = false; + multiTouch = false; + touchInProgress = false; + } + + function onMouseWheel(e) { + // if client does not want to handle this event - just ignore the call + if (beforeWheel(e)) return; + + smoothScroll.cancel(); + + var delta = e.deltaY; + if (e.deltaMode > 0) delta *= 100; + + var scaleMultiplier = getScaleMultiplier(delta); + + if (scaleMultiplier !== 1) { + var offset = transformOrigin + ? getTransformOriginOffset() + : getOffsetXY(e); + publicZoomTo(offset.x, offset.y, scaleMultiplier); + e.preventDefault(); + } + } + + function getOffsetXY(e) { + var offsetX, offsetY; + // I tried using e.offsetX, but that gives wrong results for svg, when user clicks on a path. + var ownerRect = owner.getBoundingClientRect(); + offsetX = e.clientX - ownerRect.left; + offsetY = e.clientY - ownerRect.top; + + return { x: offsetX, y: offsetY }; + } + + function smoothZoom(clientX, clientY, scaleMultiplier) { + var fromValue = transform.scale; + var from = { scale: fromValue }; + var to = { scale: scaleMultiplier * fromValue }; + + smoothScroll.cancel(); + cancelZoomAnimation(); + + zoomToAnimation = animate(from, to, { + step: function (v) { + zoomAbs(clientX, clientY, v.scale); + }, + done: triggerZoomEnd + }); + } + + function smoothZoomAbs(clientX, clientY, toScaleValue) { + var fromValue = transform.scale; + var from = { scale: fromValue }; + var to = { scale: toScaleValue }; + + smoothScroll.cancel(); + cancelZoomAnimation(); + + zoomToAnimation = animate(from, to, { + step: function (v) { + zoomAbs(clientX, clientY, v.scale); + } + }); + } + + function getTransformOriginOffset() { + var ownerRect = owner.getBoundingClientRect(); + return { + x: ownerRect.width * transformOrigin.x, + y: ownerRect.height * transformOrigin.y + }; + } + + function publicZoomTo(clientX, clientY, scaleMultiplier) { + smoothScroll.cancel(); + cancelZoomAnimation(); + return zoomByRatio(clientX, clientY, scaleMultiplier); + } + + function cancelZoomAnimation() { + if (zoomToAnimation) { + zoomToAnimation.cancel(); + zoomToAnimation = null; + } + } + + function getScaleMultiplier(delta) { + var sign = Math.sign(delta); + var deltaAdjustedSpeed = Math.min(0.25, Math.abs(speed * delta / 128)); + return 1 - sign * deltaAdjustedSpeed; + } + + function triggerPanStart() { + if (!panstartFired) { + triggerEvent('panstart'); + panstartFired = true; + smoothScroll.start(); + } + } + + function triggerPanEnd() { + if (panstartFired) { + // we should never run smooth scrolling if it was multiTouch (pinch zoom animation): + if (!multiTouch) smoothScroll.stop(); + triggerEvent('panend'); + } + } + + function triggerZoomEnd() { + triggerEvent('zoomend'); + } + + function triggerEvent(name) { + api.fire(name, api); + } +} + +function parseTransformOrigin(options) { + if (!options) return; + if (typeof options === 'object') { + if (!isNumber(options.x) || !isNumber(options.y)) + failTransformOrigin(options); + return options; + } + + failTransformOrigin(); +} + +function failTransformOrigin(options) { + console.error(options); + throw new Error( + [ + 'Cannot parse transform origin.', + 'Some good examples:', + ' "center center" can be achieved with {x: 0.5, y: 0.5}', + ' "top center" can be achieved with {x: 0.5, y: 0}', + ' "bottom right" can be achieved with {x: 1, y: 1}' + ].join('\n') + ); +} + +function noop() { } + +function validateBounds(bounds) { + var boundsType = typeof bounds; + if (boundsType === 'undefined' || boundsType === 'boolean') return; // this is okay + // otherwise need to be more thorough: + var validBounds = + isNumber(bounds.left) && + isNumber(bounds.top) && + isNumber(bounds.bottom) && + isNumber(bounds.right); + + if (!validBounds) + throw new Error( + 'Bounds object is not valid. It can be: ' + + 'undefined, boolean (true|false) or an object {left, top, right, bottom}' + ); +} + +function isNumber(x) { + return Number.isFinite(x); +} + +// IE 11 does not support isNaN: +function isNaN(value) { + if (Number.isNaN) { + return Number.isNaN(value); + } + + return value !== value; +} + +function rigidScroll() { + return { + start: noop, + stop: noop, + cancel: noop + }; +} + +function autoRun() { + if (typeof document === 'undefined') return; + + var scripts = document.getElementsByTagName('script'); + if (!scripts) return; + var panzoomScript; + + for (var i = 0; i < scripts.length; ++i) { + var x = scripts[i]; + if (x.src && x.src.match(/\bpanzoom(\.min)?\.js/)) { + panzoomScript = x; + break; + } + } + + if (!panzoomScript) return; + + var query = panzoomScript.getAttribute('query'); + if (!query) return; + + var globalName = panzoomScript.getAttribute('name') || 'pz'; + var started = Date.now(); + + tryAttach(); + + function tryAttach() { + var el = document.querySelector(query); + if (!el) { + var now = Date.now(); + var elapsed = now - started; + if (elapsed < 2000) { + // Let's wait a bit + setTimeout(tryAttach, 100); + return; + } + // If we don't attach within 2 seconds to the target element, consider it a failure + console.error('Cannot find the panzoom element', globalName); + return; + } + var options = collectOptions(panzoomScript); + console.log(options); + window[globalName] = createPanZoom(el, options); + } + + function collectOptions(script) { + var attrs = script.attributes; + var options = {}; + for (var i = 0; i < attrs.length; ++i) { + var attr = attrs[i]; + var nameValue = getPanzoomAttributeNameValue(attr); + if (nameValue) { + options[nameValue.name] = nameValue.value; + } + } + + return options; + } + + function getPanzoomAttributeNameValue(attr) { + if (!attr.name) return; + var isPanZoomAttribute = + attr.name[0] === 'p' && attr.name[1] === 'z' && attr.name[2] === '-'; + + if (!isPanZoomAttribute) return; + + var name = attr.name.substr(3); + var value = JSON.parse(attr.value); + return { name: name, value: value }; + } +} + +autoRun(); + +},{"./lib/createTextSelectionInterceptor.js":2,"./lib/domController.js":3,"./lib/kinetic.js":4,"./lib/svgController.js":5,"./lib/transform.js":6,"amator":7,"ngraph.events":9,"wheel":10}],2:[function(require,module,exports){ +/** + * Disallows selecting text. + */ +module.exports = createTextSelectionInterceptor; + +function createTextSelectionInterceptor(useFake) { + if (useFake) { + return { + capture: noop, + release: noop + }; + } + + var dragObject; + var prevSelectStart; + var prevDragStart; + var wasCaptured = false; + + return { + capture: capture, + release: release + }; + + function capture(domObject) { + wasCaptured = true; + prevSelectStart = window.document.onselectstart; + prevDragStart = window.document.ondragstart; + + window.document.onselectstart = disabled; + + dragObject = domObject; + dragObject.ondragstart = disabled; + } + + function release() { + if (!wasCaptured) return; + + wasCaptured = false; + window.document.onselectstart = prevSelectStart; + if (dragObject) dragObject.ondragstart = prevDragStart; + } +} + +function disabled(e) { + e.stopPropagation(); + return false; +} + +function noop() {} + +},{}],3:[function(require,module,exports){ +module.exports = makeDomController + +module.exports.canAttach = isDomElement; + +function makeDomController(domElement, options) { + var elementValid = isDomElement(domElement); + if (!elementValid) { + throw new Error('panzoom requires DOM element to be attached to the DOM tree') + } + + var owner = domElement.parentElement; + domElement.scrollTop = 0; + + if (!options.disableKeyboardInteraction) { + owner.setAttribute('tabindex', 0); + } + + var api = { + getBBox: getBBox, + getOwner: getOwner, + applyTransform: applyTransform, + } + + return api + + function getOwner() { + return owner + } + + function getBBox() { + // TODO: We should probably cache this? + return { + left: 0, + top: 0, + width: domElement.clientWidth, + height: domElement.clientHeight + } + } + + function applyTransform(transform) { + // TODO: Should we cache this? + domElement.style.transformOrigin = '0 0 0'; + domElement.style.transform = 'matrix(' + + transform.scale + ', 0, 0, ' + + transform.scale + ', ' + + transform.x + ', ' + transform.y + ')' + } +} + +function isDomElement(element) { + return element && element.parentElement && element.style; +} + +},{}],4:[function(require,module,exports){ +/** + * Allows smooth kinetic scrolling of the surface + */ +module.exports = kinetic; + +function kinetic(getPoint, scroll, settings) { + if (typeof settings !== 'object') { + // setting could come as boolean, we should ignore it, and use an object. + settings = {}; + } + + var minVelocity = typeof settings.minVelocity === 'number' ? settings.minVelocity : 5; + var amplitude = typeof settings.amplitude === 'number' ? settings.amplitude : 0.25; + var cancelAnimationFrame = typeof settings.cancelAnimationFrame === 'function' ? settings.cancelAnimationFrame : getCancelAnimationFrame(); + var requestAnimationFrame = typeof settings.requestAnimationFrame === 'function' ? settings.requestAnimationFrame : getRequestAnimationFrame(); + + var lastPoint; + var timestamp; + var timeConstant = 342; + + var ticker; + var vx, targetX, ax; + var vy, targetY, ay; + + var raf; + + return { + start: start, + stop: stop, + cancel: dispose + }; + + function dispose() { + cancelAnimationFrame(ticker); + cancelAnimationFrame(raf); + } + + function start() { + lastPoint = getPoint(); + + ax = ay = vx = vy = 0; + timestamp = new Date(); + + cancelAnimationFrame(ticker); + cancelAnimationFrame(raf); + + // we start polling the point position to accumulate velocity + // Once we stop(), we will use accumulated velocity to keep scrolling + // an object. + ticker = requestAnimationFrame(track); + } + + function track() { + var now = Date.now(); + var elapsed = now - timestamp; + timestamp = now; + + var currentPoint = getPoint(); + + var dx = currentPoint.x - lastPoint.x; + var dy = currentPoint.y - lastPoint.y; + + lastPoint = currentPoint; + + var dt = 1000 / (1 + elapsed); + + // moving average + vx = 0.8 * dx * dt + 0.2 * vx; + vy = 0.8 * dy * dt + 0.2 * vy; + + ticker = requestAnimationFrame(track); + } + + function stop() { + cancelAnimationFrame(ticker); + cancelAnimationFrame(raf); + + var currentPoint = getPoint(); + + targetX = currentPoint.x; + targetY = currentPoint.y; + timestamp = Date.now(); + + if (vx < -minVelocity || vx > minVelocity) { + ax = amplitude * vx; + targetX += ax; + } + + if (vy < -minVelocity || vy > minVelocity) { + ay = amplitude * vy; + targetY += ay; + } + + raf = requestAnimationFrame(autoScroll); + } + + function autoScroll() { + var elapsed = Date.now() - timestamp; + + var moving = false; + var dx = 0; + var dy = 0; + + if (ax) { + dx = -ax * Math.exp(-elapsed / timeConstant); + + if (dx > 0.5 || dx < -0.5) moving = true; + else dx = ax = 0; + } + + if (ay) { + dy = -ay * Math.exp(-elapsed / timeConstant); + + if (dy > 0.5 || dy < -0.5) moving = true; + else dy = ay = 0; + } + + if (moving) { + scroll(targetX + dx, targetY + dy); + raf = requestAnimationFrame(autoScroll); + } + } +} + +function getCancelAnimationFrame() { + if (typeof cancelAnimationFrame === 'function') return cancelAnimationFrame; + return clearTimeout; +} + +function getRequestAnimationFrame() { + if (typeof requestAnimationFrame === 'function') return requestAnimationFrame; + + return function (handler) { + return setTimeout(handler, 16); + } +} +},{}],5:[function(require,module,exports){ +module.exports = makeSvgController +module.exports.canAttach = isSVGElement; + +function makeSvgController(svgElement, options) { + if (!isSVGElement(svgElement)) { + throw new Error('svg element is required for svg.panzoom to work') + } + + var owner = svgElement.ownerSVGElement + if (!owner) { + throw new Error( + 'Do not apply panzoom to the root element. ' + + 'Use its child instead (e.g. ). ' + + 'As of March 2016 only FireFox supported transform on the root element') + } + + if (!options.disableKeyboardInteraction) { + owner.setAttribute('tabindex', 0); + } + + var api = { + getBBox: getBBox, + getScreenCTM: getScreenCTM, + getOwner: getOwner, + applyTransform: applyTransform, + initTransform: initTransform + } + + return api + + function getOwner() { + return owner + } + + function getBBox() { + var bbox = svgElement.getBBox() + return { + left: bbox.x, + top: bbox.y, + width: bbox.width, + height: bbox.height, + } + } + + function getScreenCTM() { + var ctm = owner.getCTM(); + if (!ctm) { + // This is likely firefox: https://bugzilla.mozilla.org/show_bug.cgi?id=873106 + // The code below is not entirely correct, but still better than nothing + return owner.getScreenCTM(); + } + return ctm; + } + + function initTransform(transform) { + var screenCTM = svgElement.getCTM() + + // The above line returns null on Firefox + if (screenCTM === null) { + screenCTM = document.createElementNS("http://www.w3.org/2000/svg", "svg").createSVGMatrix() + } + + transform.x = screenCTM.e; + transform.y = screenCTM.f; + transform.scale = screenCTM.a; + owner.removeAttributeNS(null, 'viewBox'); + } + + function applyTransform(transform) { + svgElement.setAttribute('transform', 'matrix(' + + transform.scale + ' 0 0 ' + + transform.scale + ' ' + + transform.x + ' ' + transform.y + ')') + } +} + +function isSVGElement(element) { + return element && element.ownerSVGElement && element.getCTM; +} +},{}],6:[function(require,module,exports){ +module.exports = Transform; + +function Transform() { + this.x = 0; + this.y = 0; + this.scale = 1; +} + +},{}],7:[function(require,module,exports){ +var BezierEasing = require('bezier-easing') + +// Predefined set of animations. Similar to CSS easing functions +var animations = { + ease: BezierEasing(0.25, 0.1, 0.25, 1), + easeIn: BezierEasing(0.42, 0, 1, 1), + easeOut: BezierEasing(0, 0, 0.58, 1), + easeInOut: BezierEasing(0.42, 0, 0.58, 1), + linear: BezierEasing(0, 0, 1, 1) +} + + +module.exports = animate; +module.exports.makeAggregateRaf = makeAggregateRaf; +module.exports.sharedScheduler = makeAggregateRaf(); + + +function animate(source, target, options) { + var start = Object.create(null) + var diff = Object.create(null) + options = options || {} + // We let clients specify their own easing function + var easing = (typeof options.easing === 'function') ? options.easing : animations[options.easing] + + // if nothing is specified, default to ease (similar to CSS animations) + if (!easing) { + if (options.easing) { + console.warn('Unknown easing function in amator: ' + options.easing); + } + easing = animations.ease + } + + var step = typeof options.step === 'function' ? options.step : noop + var done = typeof options.done === 'function' ? options.done : noop + + var scheduler = getScheduler(options.scheduler) + + var keys = Object.keys(target) + keys.forEach(function(key) { + start[key] = source[key] + diff[key] = target[key] - source[key] + }) + + var durationInMs = typeof options.duration === 'number' ? options.duration : 400 + var durationInFrames = Math.max(1, durationInMs * 0.06) // 0.06 because 60 frames pers 1,000 ms + var previousAnimationId + var frame = 0 + + previousAnimationId = scheduler.next(loop) + + return { + cancel: cancel + } + + function cancel() { + scheduler.cancel(previousAnimationId) + previousAnimationId = 0 + } + + function loop() { + var t = easing(frame/durationInFrames) + frame += 1 + setValues(t) + if (frame <= durationInFrames) { + previousAnimationId = scheduler.next(loop) + step(source) + } else { + previousAnimationId = 0 + setTimeout(function() { done(source) }, 0) + } + } + + function setValues(t) { + keys.forEach(function(key) { + source[key] = diff[key] * t + start[key] + }) + } +} + +function noop() { } + +function getScheduler(scheduler) { + if (!scheduler) { + var canRaf = typeof window !== 'undefined' && window.requestAnimationFrame + return canRaf ? rafScheduler() : timeoutScheduler() + } + if (typeof scheduler.next !== 'function') throw new Error('Scheduler is supposed to have next(cb) function') + if (typeof scheduler.cancel !== 'function') throw new Error('Scheduler is supposed to have cancel(handle) function') + + return scheduler +} + +function rafScheduler() { + return { + next: window.requestAnimationFrame.bind(window), + cancel: window.cancelAnimationFrame.bind(window) + } +} + +function timeoutScheduler() { + return { + next: function(cb) { + return setTimeout(cb, 1000/60) + }, + cancel: function (id) { + return clearTimeout(id) + } + } +} + +function makeAggregateRaf() { + var frontBuffer = new Set(); + var backBuffer = new Set(); + var frameToken = 0; + + return { + next: next, + cancel: next, + clearAll: clearAll + } + + function clearAll() { + frontBuffer.clear(); + backBuffer.clear(); + cancelAnimationFrame(frameToken); + frameToken = 0; + } + + function next(callback) { + backBuffer.add(callback); + renderNextFrame(); + } + + function renderNextFrame() { + if (!frameToken) frameToken = requestAnimationFrame(renderFrame); + } + + function renderFrame() { + frameToken = 0; + + var t = backBuffer; + backBuffer = frontBuffer; + frontBuffer = t; + + frontBuffer.forEach(function(callback) { + callback(); + }); + frontBuffer.clear(); + } + + function cancel(callback) { + backBuffer.delete(callback); + } +} + +},{"bezier-easing":8}],8:[function(require,module,exports){ +/** + * https://github.com/gre/bezier-easing + * BezierEasing - use bezier curve for transition easing function + * by Gaëtan Renaudeau 2014 - 2015 – MIT License + */ + +// These values are established by empiricism with tests (tradeoff: performance VS precision) +var NEWTON_ITERATIONS = 4; +var NEWTON_MIN_SLOPE = 0.001; +var SUBDIVISION_PRECISION = 0.0000001; +var SUBDIVISION_MAX_ITERATIONS = 10; + +var kSplineTableSize = 11; +var kSampleStepSize = 1.0 / (kSplineTableSize - 1.0); + +var float32ArraySupported = typeof Float32Array === 'function'; + +function A (aA1, aA2) { return 1.0 - 3.0 * aA2 + 3.0 * aA1; } +function B (aA1, aA2) { return 3.0 * aA2 - 6.0 * aA1; } +function C (aA1) { return 3.0 * aA1; } + +// Returns x(t) given t, x1, and x2, or y(t) given t, y1, and y2. +function calcBezier (aT, aA1, aA2) { return ((A(aA1, aA2) * aT + B(aA1, aA2)) * aT + C(aA1)) * aT; } + +// Returns dx/dt given t, x1, and x2, or dy/dt given t, y1, and y2. +function getSlope (aT, aA1, aA2) { return 3.0 * A(aA1, aA2) * aT * aT + 2.0 * B(aA1, aA2) * aT + C(aA1); } + +function binarySubdivide (aX, aA, aB, mX1, mX2) { + var currentX, currentT, i = 0; + do { + currentT = aA + (aB - aA) / 2.0; + currentX = calcBezier(currentT, mX1, mX2) - aX; + if (currentX > 0.0) { + aB = currentT; + } else { + aA = currentT; + } + } while (Math.abs(currentX) > SUBDIVISION_PRECISION && ++i < SUBDIVISION_MAX_ITERATIONS); + return currentT; +} + +function newtonRaphsonIterate (aX, aGuessT, mX1, mX2) { + for (var i = 0; i < NEWTON_ITERATIONS; ++i) { + var currentSlope = getSlope(aGuessT, mX1, mX2); + if (currentSlope === 0.0) { + return aGuessT; + } + var currentX = calcBezier(aGuessT, mX1, mX2) - aX; + aGuessT -= currentX / currentSlope; + } + return aGuessT; +} + +function LinearEasing (x) { + return x; +} + +module.exports = function bezier (mX1, mY1, mX2, mY2) { + if (!(0 <= mX1 && mX1 <= 1 && 0 <= mX2 && mX2 <= 1)) { + throw new Error('bezier x values must be in [0, 1] range'); + } + + if (mX1 === mY1 && mX2 === mY2) { + return LinearEasing; + } + + // Precompute samples table + var sampleValues = float32ArraySupported ? new Float32Array(kSplineTableSize) : new Array(kSplineTableSize); + for (var i = 0; i < kSplineTableSize; ++i) { + sampleValues[i] = calcBezier(i * kSampleStepSize, mX1, mX2); + } + + function getTForX (aX) { + var intervalStart = 0.0; + var currentSample = 1; + var lastSample = kSplineTableSize - 1; + + for (; currentSample !== lastSample && sampleValues[currentSample] <= aX; ++currentSample) { + intervalStart += kSampleStepSize; + } + --currentSample; + + // Interpolate to provide an initial guess for t + var dist = (aX - sampleValues[currentSample]) / (sampleValues[currentSample + 1] - sampleValues[currentSample]); + var guessForT = intervalStart + dist * kSampleStepSize; + + var initialSlope = getSlope(guessForT, mX1, mX2); + if (initialSlope >= NEWTON_MIN_SLOPE) { + return newtonRaphsonIterate(aX, guessForT, mX1, mX2); + } else if (initialSlope === 0.0) { + return guessForT; + } else { + return binarySubdivide(aX, intervalStart, intervalStart + kSampleStepSize, mX1, mX2); + } + } + + return function BezierEasing (x) { + // Because JavaScript number are imprecise, we should guarantee the extremes are right. + if (x === 0) { + return 0; + } + if (x === 1) { + return 1; + } + return calcBezier(getTForX(x), mY1, mY2); + }; +}; + +},{}],9:[function(require,module,exports){ +module.exports = function eventify(subject) { + validateSubject(subject); + + var eventsStorage = createEventsStorage(subject); + subject.on = eventsStorage.on; + subject.off = eventsStorage.off; + subject.fire = eventsStorage.fire; + return subject; +}; + +function createEventsStorage(subject) { + // Store all event listeners to this hash. Key is event name, value is array + // of callback records. + // + // A callback record consists of callback function and its optional context: + // { 'eventName' => [{callback: function, ctx: object}] } + var registeredEvents = Object.create(null); + + return { + on: function (eventName, callback, ctx) { + if (typeof callback !== 'function') { + throw new Error('callback is expected to be a function'); + } + var handlers = registeredEvents[eventName]; + if (!handlers) { + handlers = registeredEvents[eventName] = []; + } + handlers.push({callback: callback, ctx: ctx}); + + return subject; + }, + + off: function (eventName, callback) { + var wantToRemoveAll = (typeof eventName === 'undefined'); + if (wantToRemoveAll) { + // Killing old events storage should be enough in this case: + registeredEvents = Object.create(null); + return subject; + } + + if (registeredEvents[eventName]) { + var deleteAllCallbacksForEvent = (typeof callback !== 'function'); + if (deleteAllCallbacksForEvent) { + delete registeredEvents[eventName]; + } else { + var callbacks = registeredEvents[eventName]; + for (var i = 0; i < callbacks.length; ++i) { + if (callbacks[i].callback === callback) { + callbacks.splice(i, 1); + } + } + } + } + + return subject; + }, + + fire: function (eventName) { + var callbacks = registeredEvents[eventName]; + if (!callbacks) { + return subject; + } + + var fireArguments; + if (arguments.length > 1) { + fireArguments = Array.prototype.splice.call(arguments, 1); + } + for(var i = 0; i < callbacks.length; ++i) { + var callbackInfo = callbacks[i]; + callbackInfo.callback.apply(callbackInfo.ctx, fireArguments); + } + + return subject; + } + }; +} + +function validateSubject(subject) { + if (!subject) { + throw new Error('Eventify cannot use falsy object as events subject'); + } + var reservedWords = ['on', 'fire', 'off']; + for (var i = 0; i < reservedWords.length; ++i) { + if (subject.hasOwnProperty(reservedWords[i])) { + throw new Error("Subject cannot be eventified, since it already has property '" + reservedWords[i] + "'"); + } + } +} + +},{}],10:[function(require,module,exports){ +/** + * This module used to unify mouse wheel behavior between different browsers in 2014 + * Now it's just a wrapper around addEventListener('wheel'); + * + * Usage: + * var addWheelListener = require('wheel').addWheelListener; + * var removeWheelListener = require('wheel').removeWheelListener; + * addWheelListener(domElement, function (e) { + * // mouse wheel event + * }); + * removeWheelListener(domElement, function); + */ + +module.exports = addWheelListener; + +// But also expose "advanced" api with unsubscribe: +module.exports.addWheelListener = addWheelListener; +module.exports.removeWheelListener = removeWheelListener; + + +function addWheelListener(element, listener, useCapture) { + element.addEventListener('wheel', listener, useCapture); +} + +function removeWheelListener( element, listener, useCapture ) { + element.removeEventListener('wheel', listener, useCapture); +} +},{}]},{},[1])(1) +}); From be0bfbcd27f867a31c7ba94d39d8adada0083fd2 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 5 Jul 2023 11:00:29 -0400 Subject: [PATCH 16/42] fix samplers config --- DIFFUSERS.md | 28 +++++++++++++++++++++++----- javascript/style.css | 2 +- modules/sd_models.py | 14 +++++++------- modules/shared.py | 2 +- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index ac9dc998e..3a57dec92 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -68,12 +68,14 @@ whats implemented so far? even if extensions are not supported, runtime errors are never nice will need to handle in the code before we get out of alpha -- controlnet - `sd_model.model?.diffusion_model?` -- multi-diffusion - `sd_model.first_stage_model?.encoder?` -- lycoris +- lycoris `lyco_patch_lora` +- controlnet + > sd_model.model?.diffusion_model? +- multi-diffusion + > sd_model.first_stage_model?.encoder? +- dynamic-thresholding + > AttributeError: 'DiffusionSampler' object has no attribute 'model_wrap_cfg' ## Issues @@ -120,6 +122,8 @@ will need to handle in the code before we get out of alpha > In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag. - do you have plans to implement [Restart](https://github.com/vladmandic/automatic/issues/1537) sampler in diffusers? - `torch.nonzero()` performance issue +- `enable_sequential_cpu_offload()` results in error + > NotImplementedError: Cannot copy out of meta tensor; no data! ## Update @@ -132,3 +136,17 @@ will need to handle in the code before we get out of alpha - redid samplers - fixed "it looks like the config file at 'xxx.safetensors' is not a valid JSON file" - ui settings -> samplers is now dynamic depending if backend is original or diffusers + +## Performance + +| pipeline | performance it/s | memory cpu/gpu | +| --- | --- | --- | +| original | | | +| diffusers | 8.98 / 7.44 / 8.16 / 8.41 / 7.04 | 4.3 / 9.0 | +| diffusers with safetensors | 8.91 / 7.35 / 8.11 / 8.4 / 7.09 | 5.9 / 9.0 | +| diffusers medvram | 7.52 / 6.72 / 7.53 / 7.84 / 7.21 | 6.6 / 8.2 | +| diffusers lowvram | | | + +Notes: + +- Performance is measured for batch sizes 1, 2, 4, 8 16 diff --git a/javascript/style.css b/javascript/style.css index 6340485f2..b71b2ec8d 100644 --- a/javascript/style.css +++ b/javascript/style.css @@ -541,7 +541,7 @@ table.settings-value-table td{ .extra-networks-tab { padding: 0 !important; } .extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; max-height: 50vh; min-width: 80px; max-width: 120px; } .extra-networks-page { display: flex } -.extra-networks .custom-button { min-width: 80px; max-width: 120px; width: 100%; background: none; justify-content: left; text-align: left; padding: 2px 8px 2px 8px; box-shadow: none; line-break: anywhere; } +.extra-networks .custom-button { min-width: 80px; max-width: 120px; width: 100%; background: none; justify-content: left; text-align: left; padding: 2px 8px 2px 8px; box-shadow: none; line-break: auto; } .extra-networks .custom-button:hover { background: var(--button-primary-background-fill) } .extra-network-cards { display: flex; flex-wrap: wrap; height: 50vh; max-height: 50vh; overflow-y: scroll; overflow-x: hidden; width: -webkit-fill-available; } .extra-network-cards .card { height: fit-content; margin: 0.5em; position: relative; scroll-snap-align: start; scroll-margin-top: 0; } diff --git a/modules/sd_models.py b/modules/sd_models.py index 66df8a0ac..6ba07c52e 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -582,28 +582,28 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No prior = diffusers.DiffusionPipeline.from_pretrained(prior_id, **diffusers_load_config) sd_model = PriorPipeline(prior=prior, main=sd_model) # wrap sd_model - if hasattr(sd_model, "enable_sequential_cpu_offload"): - if shared.cmd_opts.lowvram or shared.opts.diffusers_seq_cpu_offload: - sd_model.enable_sequential_cpu_offload() - shared.log.debug('Diffusers: enable sequenctial CPU offload') if hasattr(sd_model, "enable_model_cpu_offload"): if shared.cmd_opts.medvram or shared.opts.diffusers_model_cpu_offload: shared.log.debug('Diffusers: enable model CPU offload') sd_model.enable_model_cpu_offload() + if hasattr(sd_model, "enable_sequential_cpu_offload"): + if shared.opts.diffusers_seq_cpu_offload: + sd_model.enable_sequential_cpu_offload() + shared.log.debug('Diffusers: enable sequential CPU offload') if hasattr(sd_model, "enable_vae_slicing"): - if shared.opts.diffusers_vae_slicing: + if shared.cmd_opts.lowvram or shared.opts.diffusers_vae_slicing: shared.log.debug('Diffusers: enable VAE slicing') sd_model.enable_vae_slicing() else: sd_model.disable_vae_slicing() if hasattr(sd_model, "enable_vae_tiling"): - if shared.opts.diffusers_vae_tiling: + if shared.cmd_opts.lowvram or shared.opts.diffusers_vae_tiling: shared.log.debug('Diffusers: enable VAE tiling') sd_model.enable_vae_tiling() else: sd_model.disable_vae_tiling() if hasattr(sd_model, "enable_attention_slicing"): - if shared.opts.diffusers_attention_slicing: + if shared.cmd_opts.lowvram or shared.opts.diffusers_attention_slicing: shared.log.debug('Diffusers: enable attention slicing') sd_model.enable_attention_slicing() else: diff --git a/modules/shared.py b/modules/shared.py index c9aae6b90..68945b2c7 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -472,11 +472,11 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"), "show_samplers": OptionInfo(["Euler a", "UniPC", "DEIS", "DDIM", "DPM 1S", "DPM 2M", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), "fallback_sampler": OptionInfo("Euler a", "Secondary sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), "force_latent_sampler": OptionInfo("None", "Force latent upscaler sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), + "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems"), })) if backend == Backend.ORIGINAL: options_templates.update(options_section(('sampler-params', "Sampler Settings"), { - "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems"), "enable_quantization": OptionInfo(True, "Enable samplers quantization for sharper and cleaner results"), "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), "eta_ddim": OptionInfo(0.0, "Noise multiplier for DDIM (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), From cc69c3f89fb2d39f00010c9f3187f5b766504dc8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 5 Jul 2023 11:21:02 -0400 Subject: [PATCH 17/42] init samplers config regardless of pipeline --- DIFFUSERS.md | 16 +++++++++++----- modules/shared.py | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index 3a57dec92..bd7699f97 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -141,12 +141,18 @@ will need to handle in the code before we get out of alpha | pipeline | performance it/s | memory cpu/gpu | | --- | --- | --- | -| original | | | -| diffusers | 8.98 / 7.44 / 8.16 / 8.41 / 7.04 | 4.3 / 9.0 | -| diffusers with safetensors | 8.91 / 7.35 / 8.11 / 8.4 / 7.09 | 5.9 / 9.0 | -| diffusers medvram | 7.52 / 6.72 / 7.53 / 7.84 / 7.21 | 6.6 / 8.2 | -| diffusers lowvram | | | +| original | 7.99 / 7.93 / 8.83 / 9.14 / 9.2 | 6.7 / 7.2 | +| original medvram | 6.23 / 7.16 / 8.41 / 9.24 / 9.68 | 8.4 / 6.8 | +| original lowvram | | | +| diffusers | 9 / 7.4 / 8.2 / 8.4 / 7.0 | 4.3 / 9.0 | +| diffusers medvram | 7.5 / 6.7 / 7.5 / 7.8 / 7.2 | 6.6 / 8.2 | +| diffusers lowvram | 7.0 / 7.0 / 7.4 / 7.7 / 7.8 | 4.3 / 7.2 | +| diffusers with safetensors | 8.9 / 7.3 / 8.1 / 8.4 / 7.1 | 5.9 / 9.0 | Notes: - Performance is measured for batch sizes 1, 2, 4, 8 16 +- Test environment: + - nVidia RTX 3060 GPU + - Torch 2.1-nightly with CUDA 12.1 + - Cross-optimization: SDP diff --git a/modules/shared.py b/modules/shared.py index 68945b2c7..29cb28fe0 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -472,11 +472,11 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"), "show_samplers": OptionInfo(["Euler a", "UniPC", "DEIS", "DDIM", "DPM 1S", "DPM 2M", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), "fallback_sampler": OptionInfo("Euler a", "Secondary sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), "force_latent_sampler": OptionInfo("None", "Force latent upscaler sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), - "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems"), })) if backend == Backend.ORIGINAL: options_templates.update(options_section(('sampler-params', "Sampler Settings"), { + "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems"), "enable_quantization": OptionInfo(True, "Enable samplers quantization for sharper and cleaner results"), "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), "eta_ddim": OptionInfo(0.0, "Noise multiplier for DDIM (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), @@ -494,6 +494,23 @@ if backend == Backend.ORIGINAL: })) elif backend == Backend.DIFFUSERS: options_templates.update(options_section(('sampler-params', "Sampler Settings"), { + # hidden - included for compatibility only + "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems", { "visible": False}), + "enable_quantization": OptionInfo(True, "Enable samplers quantization for sharper and cleaner results", { "visible": False}), + "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}, { "visible": False}), + "eta_ddim": OptionInfo(0.0, "Noise multiplier for DDIM (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}, { "visible": False}), + "ddim_discretize": OptionInfo('uniform', "DDIM discretize img2img", gr.Radio, {"choices": ['uniform', 'quad']}, { "visible": False}), + 's_churn': OptionInfo(0.0, "sigma churn", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}, { "visible": False}), + 's_min_uncond': OptionInfo(0, "sigma negative guidance minimum ", gr.Slider, {"minimum": 0.0, "maximum": 4.0, "step": 0.01}, { "visible": False}), + 's_tmin': OptionInfo(0.0, "sigma tmin", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}, { "visible": False}), + 's_noise': OptionInfo(1.0, "sigma noise", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}, { "visible": False}), + 'eta_noise_seed_delta': OptionInfo(0, "Noise seed delta (eta)", gr.Number, {"precision": 0}, { "visible": False}), + 'always_discard_next_to_last_sigma': OptionInfo(False, "Always discard next-to-last sigma", { "visible": False}), + 'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}, { "visible": False}), + 'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}, { "visible": False}), + 'uni_pc_order': OptionInfo(3, "UniPC order (must be < sampling steps)", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}, { "visible": False}), + 'uni_pc_lower_order_final': OptionInfo(True, "UniPC lower order final", { "visible": False}), + # diffuser specific "schedulers_prediction_type": OptionInfo("default", "Samplers override model prediction type", gr.Radio, lambda: {"choices": ['default', 'epsilon', 'sample', 'v-prediction']}), "schedulers_beta_schedule": OptionInfo("default", "Samplers override beta schedule", gr.Radio, lambda: {"choices": ['default', 'linear', 'scaled_linear', 'squaredcos_cap_v2']}), "schedulers_solver_order": OptionInfo(2, "Samplers solver order where applicable", gr.Slider, {"minimum": 1, "maximum": 5, "step": 1}), From d30a55e5239e875c03b90c9f987053b4aac2ed0c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 5 Jul 2023 11:54:49 -0400 Subject: [PATCH 18/42] fix settings again --- DIFFUSERS.md | 10 +++++----- modules/shared.py | 28 ++++++++++++++-------------- modules/ui.py | 8 ++++++-- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index bd7699f97..79eeb951e 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -59,9 +59,8 @@ whats implemented so far? ## Todo - sdxl model -- new schedulers -- settings -> schedulers - no idea if sd21 works out-of-the-box +- hires fix? ## Limitations @@ -143,15 +142,16 @@ will need to handle in the code before we get out of alpha | --- | --- | --- | | original | 7.99 / 7.93 / 8.83 / 9.14 / 9.2 | 6.7 / 7.2 | | original medvram | 6.23 / 7.16 / 8.41 / 9.24 / 9.68 | 8.4 / 6.8 | -| original lowvram | | | +| original lowvram | 1.05 / 1.94 / 3.2 / 4.81 / 6.46 | 8.8 / 5.2 | +| original compile inductor | | | | diffusers | 9 / 7.4 / 8.2 / 8.4 / 7.0 | 4.3 / 9.0 | | diffusers medvram | 7.5 / 6.7 / 7.5 / 7.8 / 7.2 | 6.6 / 8.2 | | diffusers lowvram | 7.0 / 7.0 / 7.4 / 7.7 / 7.8 | 4.3 / 7.2 | | diffusers with safetensors | 8.9 / 7.3 / 8.1 / 8.4 / 7.1 | 5.9 / 9.0 | - +| diffusers compile | | | Notes: -- Performance is measured for batch sizes 1, 2, 4, 8 16 +- Performance is measured for `batch-size` 1, 2, 4, 8 16 - Test environment: - nVidia RTX 3060 GPU - Torch 2.1-nightly with CUDA 12.1 diff --git a/modules/shared.py b/modules/shared.py index 29cb28fe0..e02fadac3 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -495,21 +495,21 @@ if backend == Backend.ORIGINAL: elif backend == Backend.DIFFUSERS: options_templates.update(options_section(('sampler-params', "Sampler Settings"), { # hidden - included for compatibility only - "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems", { "visible": False}), - "enable_quantization": OptionInfo(True, "Enable samplers quantization for sharper and cleaner results", { "visible": False}), - "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}, { "visible": False}), - "eta_ddim": OptionInfo(0.0, "Noise multiplier for DDIM (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}, { "visible": False}), - "ddim_discretize": OptionInfo('uniform', "DDIM discretize img2img", gr.Radio, {"choices": ['uniform', 'quad']}, { "visible": False}), - 's_churn': OptionInfo(0.0, "sigma churn", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}, { "visible": False}), - 's_min_uncond': OptionInfo(0, "sigma negative guidance minimum ", gr.Slider, {"minimum": 0.0, "maximum": 4.0, "step": 0.01}, { "visible": False}), - 's_tmin': OptionInfo(0.0, "sigma tmin", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}, { "visible": False}), - 's_noise': OptionInfo(1.0, "sigma noise", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}, { "visible": False}), + "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems", gr.Checkbox, { "visible": False}), + "enable_quantization": OptionInfo(True, "Enable samplers quantization for sharper and cleaner results", gr.Checkbox, { "visible": False}), + "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Number, { "visible": False}), + "eta_ddim": OptionInfo(0.0, "Noise multiplier for DDIM (eta)", gr.Number, { "visible": False}), + "ddim_discretize": OptionInfo('uniform', "", gr.Text, { "visible": False}), + 's_churn': OptionInfo(0.0, "sigma churn", gr.Number, { "visible": False}), + 's_min_uncond': OptionInfo(0, "sigma negative guidance minimum ", gr.Number, { "visible": False}), + 's_tmin': OptionInfo(0.0, "sigma tmin", gr.Number, { "visible": False}), + 's_noise': OptionInfo(1.0, "sigma noise", gr.Number, { "visible": False}), 'eta_noise_seed_delta': OptionInfo(0, "Noise seed delta (eta)", gr.Number, {"precision": 0}, { "visible": False}), - 'always_discard_next_to_last_sigma': OptionInfo(False, "Always discard next-to-last sigma", { "visible": False}), - 'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}, { "visible": False}), - 'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}, { "visible": False}), - 'uni_pc_order': OptionInfo(3, "UniPC order (must be < sampling steps)", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}, { "visible": False}), - 'uni_pc_lower_order_final': OptionInfo(True, "UniPC lower order final", { "visible": False}), + 'always_discard_next_to_last_sigma': OptionInfo(False, "Always discard next-to-last sigma", gr.Checkbox, { "visible": False}), + #'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}, { "visible": False}), + #'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}, { "visible": False}), + #'uni_pc_order': OptionInfo(3, "UniPC order (must be < sampling steps)", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}, { "visible": False}), + #'uni_pc_lower_order_final': OptionInfo(True, "UniPC lower order final", { "visible": False}), # diffuser specific "schedulers_prediction_type": OptionInfo("default", "Samplers override model prediction type", gr.Radio, lambda: {"choices": ['default', 'epsilon', 'sample', 'v-prediction']}), "schedulers_beta_schedule": OptionInfo("default", "Samplers override beta schedule", gr.Radio, lambda: {"choices": ['default', 'linear', 'scaled_linear', 'squaredcos_cap_v2']}), diff --git a/modules/ui.py b/modules/ui.py index 42378732b..36c99b559 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -903,9 +903,13 @@ def create_ui(startup_timer): res = comp(label=info.label, value=fun(), elem_id=elem_id, **(args or {})) create_refresh_button(res, info.refresh, info.component_args, f"refresh_{key}") else: - res = comp(label=info.label, value=fun(), elem_id=elem_id, **(args or {})) + try: + res = comp(label=info.label, value=fun(), elem_id=elem_id, **(args or {})) + except Exception as e: + modules.shared.log.error(f'Error creating setting: {key} {e}') + res = None - if not is_quicksettings: + if res is not None and not is_quicksettings: res.change(fn=None, inputs=res, _js=f'(val) => markIfModified("{key}", val)') dirty_indicator.click(fn=lambda: getattr(opts, key), outputs=res, show_progress=False) dirtyable_setting.__exit__() From 993de932ab50f3b3a6e41c76e86a38b49bf761a8 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Thu, 6 Jul 2023 01:35:00 +0900 Subject: [PATCH 19/42] Add an opts override for DirectML. --- extensions-builtin/sd-extension-system-info | 2 +- modules/dml/__init__.py | 5 ++++- modules/dml/opts.py | 5 +++++ 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 modules/dml/opts.py diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index fcdd10c79..b30e32455 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit fcdd10c7957f85504a4511f2040ec7e01f2054a8 +Subproject commit b30e324552517e68012f2487cf7b0be43616a4cc diff --git a/modules/dml/__init__.py b/modules/dml/__init__.py index 3dc89173a..9e721591c 100644 --- a/modules/dml/__init__.py +++ b/modules/dml/__init__.py @@ -3,6 +3,7 @@ import torch import torch_directml # pylint: disable=import-error import modules.dml.hijack import modules.dml.amp as amp +from modules.dml.opts import override_opts from .optimizer.unknown import UnknownOptimizer @@ -43,6 +44,8 @@ class DirectML(): DirectML._is_autocast_enabled = enabled -# Alternative of torch.cuda for DirectML. DirectML.amp = amp +# Alternative of torch.cuda for DirectML. torch.dml = DirectML + +override_opts() diff --git a/modules/dml/opts.py b/modules/dml/opts.py new file mode 100644 index 000000000..32e24a4d5 --- /dev/null +++ b/modules/dml/opts.py @@ -0,0 +1,5 @@ +from modules import shared + +def override_opts(): + if shared.cmd_opts.backend.lower() == "diffusers": + shared.opts.diffusers_generator_device = "cpu" # DirectML does not support torch.Generator API. From a076ff1b4332e136b6e2be967b33d6915e75167b Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Thu, 6 Jul 2023 01:45:12 +0900 Subject: [PATCH 20/42] Use backend enum on diffusers check. --- extensions-builtin/a1111-sd-webui-lycoris | 2 +- modules/dml/opts.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-builtin/a1111-sd-webui-lycoris b/extensions-builtin/a1111-sd-webui-lycoris index 025dea967..123d1da15 160000 --- a/extensions-builtin/a1111-sd-webui-lycoris +++ b/extensions-builtin/a1111-sd-webui-lycoris @@ -1 +1 @@ -Subproject commit 025dea96720197dd4486a5bb8e2f4d72a95a3088 +Subproject commit 123d1da15d802823480f8020312ce449523f10e2 diff --git a/modules/dml/opts.py b/modules/dml/opts.py index 32e24a4d5..59fe392a0 100644 --- a/modules/dml/opts.py +++ b/modules/dml/opts.py @@ -1,5 +1,5 @@ from modules import shared def override_opts(): - if shared.cmd_opts.backend.lower() == "diffusers": + if shared.backend == shared.Backend.DIFFUSERS: shared.opts.diffusers_generator_device = "cpu" # DirectML does not support torch.Generator API. From c91b052fc8f722697a25a40b2b973a9fe9fb26cf Mon Sep 17 00:00:00 2001 From: GalaxyTimeMachine <52193044+GalaxyTimeMachine@users.noreply.github.com> Date: Wed, 5 Jul 2023 21:15:51 +0200 Subject: [PATCH 21/42] Update ui_extensions.py Corrected typo on line 20: avilable > available --- modules/ui_extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui_extensions.py b/modules/ui_extensions.py index f5df622f5..519dcbb64 100644 --- a/modules/ui_extensions.py +++ b/modules/ui_extensions.py @@ -17,7 +17,7 @@ sort_ordering = { "default": (True, lambda x: x.get('sort_default', '')), "user extensions": (True, lambda x: x.get('sort_user', '')), "trending": (True, lambda x: x.get('sort_trending', -1)), - "update avilable": (True, lambda x: x.get('sort_update', '')), + "update available": (True, lambda x: x.get('sort_update', '')), "updated date": (True, lambda x: x.get('updated', '2000-01-01T00:00')), "created date": (True, lambda x: x.get('created', '2000-01-01T00:00')), "name": (False, lambda x: x.get('name', '').lower()), From de94eb15894977bcb24ff120798d82b1ecd1d8b8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 5 Jul 2023 18:06:30 -0400 Subject: [PATCH 22/42] api update --- DIFFUSERS.md | 4 ++-- cli/sdapi.py | 2 +- extensions-builtin/sd-extension-system-info | 2 +- modules/api/api.py | 8 ++++---- modules/ui.py | 5 ++++- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index 79eeb951e..f18e434b9 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -123,6 +123,7 @@ will need to handle in the code before we get out of alpha - `torch.nonzero()` performance issue - `enable_sequential_cpu_offload()` results in error > NotImplementedError: Cannot copy out of meta tensor; no data! +- diffusers support `xformers`, but i don't see any notes on `sdp`? ## Update @@ -143,12 +144,11 @@ will need to handle in the code before we get out of alpha | original | 7.99 / 7.93 / 8.83 / 9.14 / 9.2 | 6.7 / 7.2 | | original medvram | 6.23 / 7.16 / 8.41 / 9.24 / 9.68 | 8.4 / 6.8 | | original lowvram | 1.05 / 1.94 / 3.2 / 4.81 / 6.46 | 8.8 / 5.2 | -| original compile inductor | | | | diffusers | 9 / 7.4 / 8.2 / 8.4 / 7.0 | 4.3 / 9.0 | | diffusers medvram | 7.5 / 6.7 / 7.5 / 7.8 / 7.2 | 6.6 / 8.2 | | diffusers lowvram | 7.0 / 7.0 / 7.4 / 7.7 / 7.8 | 4.3 / 7.2 | | diffusers with safetensors | 8.9 / 7.3 / 8.1 / 8.4 / 7.1 | 5.9 / 9.0 | -| diffusers compile | | | + Notes: - Performance is measured for `batch-size` 1, 2, 4, 8 16 diff --git a/cli/sdapi.py b/cli/sdapi.py index 3d2d99ab8..3b010e663 100755 --- a/cli/sdapi.py +++ b/cli/sdapi.py @@ -177,7 +177,7 @@ def get_log(): def get_info(): import time t0 = time.time() - res = getsync('/sdapi/v1/system-info/status') + res = getsync('/sdapi/v1/system-info/status?full=true&refresh=true') t1 = time.time() print({ 'duration': 1000 * round(t1-t0, 3), **res }) return res diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index b30e32455..5b13bfeee 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit b30e324552517e68012f2487cf7b0be43616a4cc +Subproject commit 5b13bfeeebee1fc984bfde4e3171b31e4eee5a6b diff --git a/modules/api/api.py b/modules/api/api.py index 2505b0891..144d5e82e 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -237,7 +237,7 @@ class Api: script_runner = scripts.scripts_txt2img if not script_runner.scripts: script_runner.initialize_scripts(False) - ui.create_ui() + ui.create_ui(None) if not self.default_script_arg_txt2img: self.default_script_arg_txt2img = self.init_default_script_args(script_runner) selectable_scripts, selectable_script_idx = self.get_selectable_script(txt2imgreq.script_name, script_runner) @@ -282,7 +282,7 @@ class Api: script_runner = scripts.scripts_img2img if not script_runner.scripts: script_runner.initialize_scripts(True) - ui.create_ui() + ui.create_ui(None) if not self.default_script_arg_img2img: self.default_script_arg_img2img = self.init_default_script_args(script_runner) selectable_scripts, selectable_script_idx = self.get_selectable_script(img2imgreq.script_name, script_runner) @@ -650,8 +650,8 @@ class Api: "port": shared.cmd_opts.port, "keyfile": shared.cmd_opts.tls_keyfile, "certfile": shared.cmd_opts.tls_certfile, - "loop": "auto", - "http": "auto", + "loop": "auto", # auto, asyncio, uvloop + "http": "auto", # auto, h11, httptools } from modules.server import UvicornServer server = UvicornServer(self.app, **config) diff --git a/modules/ui.py b/modules/ui.py index 36c99b559..32c94c98c 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -321,7 +321,10 @@ def create_override_settings_dropdown(tabname, row): # pylint: disable=unused-ar return dropdown -def create_ui(startup_timer): +def create_ui(startup_timer = None): + if startup_timer is None: + from modules import timer + startup_timer = timer.Timer() import modules.img2img # pylint: disable=redefined-outer-name import modules.txt2img # pylint: disable=redefined-outer-name reload_javascript() From dd4602fd645865b497e558096d98ef8774b87f3e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 5 Jul 2023 18:58:23 -0400 Subject: [PATCH 23/42] update dynamo logging --- modules/sd_hijack.py | 6 ++++-- modules/sd_models.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 5464cee44..8d78a3adb 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -180,8 +180,10 @@ class StableDiffusionModelHijack: shared.log.warning("Model compile skipped: IPEX Method is for Intel GPU's with OneAPI") elif opts.cuda_compile and opts.cuda_compile_mode != 'none' and shared.backend == shared.Backend.ORIGINAL: try: - import torch._dynamo as dynamo # pylint: disable=unused-import - # torch._dynamo.config.log_level = logging.WARNING if opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access + import logging + import torch._dynamo # pylint: disable=unused-import + log_level = logging.WARNING if opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access + torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access torch._dynamo.config.verbose = opts.cuda_compile_verbose # pylint: disable=protected-access torch._dynamo.config.suppress_errors = opts.cuda_compile_errors # pylint: disable=protected-access torch.backends.cudnn.benchmark = True diff --git a/modules/sd_models.py b/modules/sd_models.py index 6ba07c52e..aad51403b 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -615,7 +615,9 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.unet.to(memory_format=torch.channels_last) if shared.opts.cuda_compile and torch.cuda.is_available(): sd_model.to(devices.device) - import torch._dynamo as dynamo # pylint: disable=unused-import + import torch._dynamo # pylint: disable=unused-import + log_level = logging.WARNING if shared.opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access + torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access torch._dynamo.config.verbose = shared.opts.cuda_compile_verbose # pylint: disable=protected-access torch._dynamo.config.suppress_errors = shared.opts.cuda_compile_errors # pylint: disable=protected-access sd_model.unet = torch.compile(sd_model.unet, mode=shared.opts.cuda_compile_mode, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init From 849877ec50b2191d8b3b4f782b536b25c4000f49 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jul 2023 08:50:38 -0400 Subject: [PATCH 24/42] version lock pillow --- DIFFUSERS.md | 65 +++-------------------------------------------- TODO.md | 1 + modules/images.py | 2 +- requirements.txt | 2 +- 4 files changed, 7 insertions(+), 63 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index f18e434b9..96d36d180 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -76,67 +76,6 @@ will need to handle in the code before we get out of alpha - dynamic-thresholding > AttributeError: 'DiffusionSampler' object has no attribute 'model_wrap_cfg' -## Issues - -- new dependency hell (not diffuser related)? - -## Notes for HF - -- removed `quicksettings` alternative completely -- added simple model downloader in ui: *tabs -> models -> huggingface* -- attempting to download gated model without access token results in model/refs/commits not found instead of access denied - this is not very user friendly, it should be handled in the code -- redone diffuser sampler support, new code in `modules/sd_samplers_diffusers.py` -- new config section ui settings -> samplers - (its dynamic, it will show standard ui or diffuser specific stuff depending how sdnext is started) -- scheduler config is a bit difficult to work with as its not possible to see which params each scheduler defines ahead of time - and if passing params it doesn't have, it will result in runtime error -- redone **textual inversion** support, core is now in `modules/textual_inversion/textual_inversion.py:load_diffusers_embedding()` - the point is that sdnext pre-loads all compatible embeddings on model load so they are available in prompt context -- redone **lora** support, core is now in `modules/lora_diffusers.py` -- added support for diffuser models in **safetensors/ckpt** format -- when i use `diffusers.StableDiffusionPipeline.from_ckpt` - first time it downloads something - what is that? (could it be a default safety checker?) - > Downloading (…)lve/main/config.json: 4.55k - > Downloading pytorch_model.bin: 1.22G -- loading safetensors model is very slow - for example, 2sec without diffusers and 16sec with diffusers -- in `modules/modelloader.py:download_diffusers_model()` i get unknown property for `hf.model_info(hub_id).cardData` - can you double-check if this is linter issue or actual problem? -- question on `pipe.load_lora_weights` - does it support loading multiple loras? i don't see any notes on that in docs - also, lora strength is specified using `cross_attention_kwargs={"scale": x}` during pipeline execution - which means if there are multiple loras, they all have the same strength? -- any plans to support more complex loras? from limited testing - it seems only basic loras are working while lycoris/locon are not -- **deepfloyd** failures: - > /home/disty/Apps/automatic/venv/lib/python3.10/site-packages/diffusers/configuration_utils.py:138 in __getattr__ - > AttributeError: 'DDPMScheduler' object has no attribute 'name -- question how do diffusers handle standard 75 token limit for sd? -- diffusers `convert_from_ckpt.py` uses fixed `print` statements so its not possible to control its output to console - it should use `logging` instead. in general, using `print` is bad idea - for example, it very annoyingly logs this every time `StableDiffusionPipeline.from_ckpt` is used: - > global_step key not found in model - > Checkpoint /home/vlado/dev/automatic/models/Stable-diffusion/best/absolutereality_v1.safetensors has both EMA and non-EMA weights. - > In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag. -- do you have plans to implement [Restart](https://github.com/vladmandic/automatic/issues/1537) sampler in diffusers? -- `torch.nonzero()` performance issue -- `enable_sequential_cpu_offload()` results in error - > NotImplementedError: Cannot copy out of meta tensor; no data! -- diffusers support `xformers`, but i don't see any notes on `sdp`? - -## Update - -- sortable models table in downloader ui -- system info tab -> benchmark is now working -- recommended scheduler: `deis` -- `channels_last` and `cudnn_benchmark` now apply to diffusers -- new settings section for diffusers fine-tuning -- fixed missed call to `devices.set_cuda_params` -- redid samplers -- fixed "it looks like the config file at 'xxx.safetensors' is not a valid JSON file" -- ui settings -> samplers is now dynamic depending if backend is original or diffusers - ## Performance | pipeline | performance it/s | memory cpu/gpu | @@ -156,3 +95,7 @@ Notes: - nVidia RTX 3060 GPU - Torch 2.1-nightly with CUDA 12.1 - Cross-optimization: SDP +- All being equal, diffussers seem to: + - Use slightly less RAM and more VRAM + - Have highly efficient medvram/lowvram equivalents which don't loose a lot of performance + - Faster on smaller batch sizes, slower on larger batch sizes diff --git a/TODO.md b/TODO.md index 60ced2f41..f4da3031c 100644 --- a/TODO.md +++ b/TODO.md @@ -60,3 +60,4 @@ Tech that can be integrated as part of the core workflow... - git-rebasin - additional upscalers - new image browser +- fp8 diff --git a/modules/images.py b/modules/images.py index efa1f7a3d..e0c8b4bbb 100644 --- a/modules/images.py +++ b/modules/images.py @@ -133,7 +133,7 @@ def draw_grid_annotations(im, width, height, hor_texts, ver_texts, margin=0): except Exception: return ImageFont.truetype('html/roboto.ttf', fontsize) - def draw_texts(drawing, draw_x, draw_y, lines, initial_fnt, initial_fontsize): + def draw_texts(drawing: ImageDraw, draw_x, draw_y, lines, initial_fnt, initial_fontsize): for line in lines: fnt = initial_fnt fontsize = initial_fontsize diff --git a/requirements.txt b/requirements.txt index fced77fd2..cdd498173 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,6 @@ omegaconf open-clip-torch opencv-contrib-python piexif -Pillow psutil pyyaml realesrgan @@ -63,3 +62,4 @@ transformers==4.26.1 timm==0.6.13 tomesd==0.1.3 urllib3==1.26.15 +Pillow==9.5.0 From d8748fd7ebdaf93cc8219d54f8cf70358cd814f8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jul 2023 09:44:43 -0400 Subject: [PATCH 25/42] theme update --- DIFFUSERS.md | 2 ++ installer.py | 2 ++ javascript/black-orange.css | 40 +++++++++++++++---------------------- javascript/style.css | 8 +------- modules/ui.py | 7 +------ 5 files changed, 22 insertions(+), 37 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index 96d36d180..d5a99b5db 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -61,6 +61,7 @@ whats implemented so far? - sdxl model - no idea if sd21 works out-of-the-box - hires fix? +- vae support ## Limitations @@ -75,6 +76,7 @@ will need to handle in the code before we get out of alpha > sd_model.first_stage_model?.encoder? - dynamic-thresholding > AttributeError: 'DiffusionSampler' object has no attribute 'model_wrap_cfg' +- no per-step callback ## Performance diff --git a/installer.py b/installer.py index 220d20e93..0ebcc0680 100644 --- a/installer.py +++ b/installer.py @@ -402,6 +402,8 @@ def install_packages(): # install(openclip_package, 'open-clip-torch') clip_package = os.environ.get('CLIP_PACKAGE', "git+https://github.com/openai/CLIP.git") install(clip_package, 'clip') + invisiblewatermark_package = os.environ.get('INVISIBLEWATERMARK_PACKAGE', "git+https://github.com/patrickvonplaten/invisible-watermark.git@remove_onnxruntime_depedency") + install(invisiblewatermark_package, 'invisible-watermark') install('onnxruntime==1.15.1', 'onnxruntime', ignore=True) if args.profile: print_profile(pr, 'Packages') diff --git a/javascript/black-orange.css b/javascript/black-orange.css index 6f07773f7..22c0e57c6 100644 --- a/javascript/black-orange.css +++ b/javascript/black-orange.css @@ -14,30 +14,24 @@ input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111; border: 0 ::-webkit-scrollbar-thumb { background-color: var(--highlight-color); border-radius: 2px; border-width: 0; box-shadow: 2px 2px 3px #111111; } div.form { border-width: 0; box-shadow: none; background: transparent; overflow: visible; gap: 0.5em; margin-bottom: 6px; } -/* main gradio components by selector */ -div.gradio-container.dark > div.w-full.flex.flex-col.min-h-screen > div { background-color: black; } - /* gradio shadowroot */ .gradio-container { font-family: var(--font); --left-column: 490px; --highlight-color: #CE6400; --inactive-color: #4E1400; } + /* gradio style classes */ +fieldset .gr-block.gr-box, label.block span { padding: 0; margin-top: -4px; } .border-2 { border-width: 0; } .border-b-2 { border-bottom-width: 2px; border-color: var(--highlight-color) !important; padding-bottom: 2px; margin-bottom: 8px; } -.dark .bg-white { color: lightyellow; border-radius: 0; background-color: var(--inactive-color); } -.dark .bg-gray-200, .dark .\!bg-gray-200 { background-color: transparent; } -.dark .dark\:bg-gray-900 { background-color: black; } -.dark .gr-box { border-radius: 0 !important; background-color: #111111 !important; box-shadow: 2px 2px 3px #111111; border-width: 0; padding: 4px; margin: 12px 0px 12px 0px } -.dark .gr-button { border-radius: 0; font-weight: normal; box-shadow: 2px 2px 3px #111111; font-size: 0.8rem; min-width: 32px; min-height: 32px; padding: 3px; margin: 3px; } -.dark .gr-check-radio { background-color: var(--inactive-color); border-width: 0; border-radius: 2px; box-shadow: 2px 2px 3px #111111; } -.dark .gr-check-radio:checked { background-color: var(--highlight-color); } -.dark .gr-compact { border-radius: 0; background-color: black; } -.dark .gr-form { border-radius: 0; border-width: 0; } -.dark .gr-input { background-color: #333333 !important; padding: 4px; margin: 4px; } -.dark .gr-input-label { color: lightyellow; border-width: 0; background: transparent; padding: 2px !important; } -.dark .gr-panel { border-radius: 0; background-color: black; } -.dark { background-color: black; } -.dark fieldset span.text-gray-500, .dark .gr-block.gr-box span.text-gray-500, .dark label.block span { padding: 0; margin-top: -4px; } -.dark fieldset span.text-gray-500, .dark .gr-block.gr-box span.text-gray-500, .dark label.block span { border-radius: 0;} +.bg-white { color: lightyellow; background-color: var(--inactive-color); } +.gr-box { border-radius: 0 !important; background-color: #111111 !important; box-shadow: 2px 2px 3px #111111; border-width: 0; padding: 4px; margin: 12px 0px 12px 0px } +.gr-button { font-weight: normal; box-shadow: 2px 2px 3px #111111; font-size: 0.8rem; min-width: 32px; min-height: 32px; padding: 3px; margin: 3px; } +.gr-check-radio { background-color: var(--inactive-color); border-width: 0; border-radius: 2px; box-shadow: 2px 2px 3px #111111; } +.gr-check-radio:checked { background-color: var(--highlight-color); } +.gr-compact { background-color: black; } +.gr-form { border-width: 0; } +.gr-input { background-color: #333333 !important; padding: 4px; margin: 4px; } +.gr-input-label { color: lightyellow; border-width: 0; background: transparent; padding: 2px !important; } +.gr-panel { background-color: black; } .eta-bar { display: none !important } svg.feather.feather-image, .feather .feather-image { display: none } .gap-2 { padding-top: 8px; } @@ -48,10 +42,8 @@ svg.feather.feather-image, .feather .feather-image { display: none } .p-2 { padding: 0; } .px-4 { padding-lefT: 1rem; padding-right: 1rem; } .py-6 { padding-bottom: 0; } -.rounded-lg { border-radius: 0; } .tabs { background-color: black; } -.gradio-button.tool { border-radius: 0; } -.block.token-counter span { background-color: #222 !important; box-shadow: 2px 2px 2px #111; border: none !important; border-radius: 0; font-size: 0.8rem; } +.block.token-counter span { background-color: #222 !important; box-shadow: 2px 2px 2px #111; border: none !important; font-size: 0.8rem; } .tab-nav { zoom: 120%; margin-bottom: 10px; border-bottom: 2px solid #CE6400 !important; padding-bottom: 2px; } .label-wrap { margin: 16px 0px 8px 0px; } .gradio-slider input[type="number"] { width: 4.5em; font-size: 0.8rem; } @@ -68,7 +60,7 @@ svg.feather.feather-image, .feather .feather-image { display: none } .image-buttons { gap: 10px !important} /* gradio elements overrides */ -#div.gradio-container.dark { overflow-x: hidden; } +#div.gradio-container { overflow-x: hidden; } #img2img_label_copy_to_img2img { font-weight: normal; } #txt2img_prompt, #txt2img_neg_prompt, #img2img_prompt, #img2img_neg_prompt { background-color: black; box-shadow: 4px 4px 4px 0px #333333 !important; } #txt2img_prompt > label > textarea, #txt2img_neg_prompt > label > textarea, #img2img_prompt > label > textarea, #img2img_neg_prompt > label > textarea { font-size: 1.2rem; } @@ -104,7 +96,7 @@ svg.feather.feather-image, .feather .feather-image { display: none } #steps-animation, #controlnet { border-width: 0; } /* based on gradio built-in dark theme */ -.dark { +:root, .light, .dark { --body-background-fill: black; --body-text-color: var(--neutral-100); --color-accent-soft: var(--neutral-700); @@ -282,7 +274,7 @@ svg.feather.feather-image, .feather .feather-image { display: none } --section-header-text-weight: 400; --checkbox-border-radius: var(--radius-sm); --checkbox-label-gap: 2px; - --checkbox-label-padding: var(--spacing-md) calc(2 * var(--spacing-md)); + --checkbox-label-padding: var(--spacing-md); --checkbox-label-shadow: var(--shadow-drop); --checkbox-label-text-size: var(--text-md); --checkbox-label-text-weight: 400; diff --git a/javascript/style.css b/javascript/style.css index b71b2ec8d..1a09def29 100644 --- a/javascript/style.css +++ b/javascript/style.css @@ -28,7 +28,7 @@ div.gradio-html.min{ min-height: 0; } footer { display: none; } /* general styled components */ -.gradio-button.tool{ max-width: 2.3em; min-width: 2.3em !important; height: 2.3em; align-self: end; line-height: 1em; border-radius: 0.5em; } +.gradio-button.tool{ max-width: 2.3em; min-width: 2.3em !important; height: 2.3em; align-self: end; line-height: 1em } .gradio-button.secondary-down{ background: var(--button-secondary-background-fill); color: var(--button-secondary-text-color); } .gradio-button.secondary-down, .gradio-button.secondary-down:hover{ box-shadow: 1px 1px 1px rgba(0,0,0,0.25) inset, 0px 0px 3px rgba(0,0,0,0.15) inset; } .gradio-button.secondary-down:hover{ background: var(--button-secondary-background-fill-hover); color: var(--button-secondary-text-color-hover); } @@ -69,7 +69,6 @@ button.custom-button{ background: var(--input-background-fill) !important; box-shadow: 0 0 0.0 0.3em rgba(192,192,192,0.15), inset 0 0 0.6em rgba(192,192,192,0.075); border: 2px solid rgba(192,192,192,0.4) !important; - border-radius: 0.4em; } .block.token-counter.error span{ @@ -316,7 +315,6 @@ div#extras_scale_to_tab div.form{ position: relative; height: 20px; background: #b4c0cc; - border-radius: 3px !important; margin-bottom: -3px; } @@ -333,7 +331,6 @@ div#extras_scale_to_tab div.form{ line-height: 20px; padding: 0 8px 0 0; text-align: right; - border-radius: 3px; overflow: visible; white-space: nowrap; padding: 0 0.5em; @@ -464,14 +461,12 @@ table.settings-value-table td{ font-weight: bold; font-size: 20px; transition: 0.6s ease; - border-radius: 0 3px 3px 0; user-select: none; -webkit-user-select: none; } .modalNext { right: 0; - border-radius: 3px 0 0 3px; } .modalPrev:hover, .modalNext:hover { @@ -497,7 +492,6 @@ table.settings-value-table td{ display:block; padding:0px 0; border:2px solid #a55000; - border-radius:8px; box-shadow:1px 1px 2px #CE6400; width: 200px; } diff --git a/modules/ui.py b/modules/ui.py index 32c94c98c..60ccc6cd1 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -1188,12 +1188,7 @@ def html_body(): body = '' inline = '' if opts.theme_style != 'Auto': - if opts.gradio_theme == 'black-orange': - modules.shared.log.info('Theme does not support custom mode') - else: - inline += f"set_theme('{opts.theme_style.lower()}');" - if opts.gradio_theme == 'black-orange': - inline += "set_theme('dark');" + inline += f"set_theme('{opts.theme_style.lower()}');" body += f'\n' return body From 7e11ff2b341cb94880b3bd2f94a9f31ef3127ba1 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jul 2023 19:26:43 -0400 Subject: [PATCH 26/42] add sdxl support --- DIFFUSERS.md | 213 +++++++++++------- installer.py | 5 +- javascript/black-orange.css | 61 ++--- modules/api/api.py | 6 +- modules/processing.py | 72 ++++-- modules/sd_models.py | 181 +++++++++++---- modules/shared.py | 25 +- .../textual_inversion/textual_inversion.py | 8 +- modules/ui.py | 3 +- webui.py | 14 +- wiki | 2 +- 11 files changed, 390 insertions(+), 200 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index d5a99b5db..61ff1c17b 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -1,4 +1,129 @@ -# Diffusers WiP +# Additional Models + +SD.Next includes *experimental* support for additional model pipelines +This includes support for additional models such as: + +- **Stable Diffusion XL** +- **Kandinsky** +- **Deep Floyd IF** +- **Shap-E** + +Note that support is *experimental*, do not open [GitHub issues](https://github.com/vladmandic/automatic/issues) for those models +and instead reach-out on [Discord](https://discord.gg/WqMzTUDC) using dedicated channels + +*This has been made possible by integration of [huggingface diffusers](https://huggingface.co/docs/diffusers/index) library with help of huggingface team!* + +## How to + +- Install **SD.Next** as usual +- Start with + `webui --backend diffusers` +- To go back to standard execution pipeline, start with + `webui --backend original` + +## Integration + +### Standard workflows + +- **txt2txt** +- **img2img** +- **process** + +### Model Access + +- For standard SD 1.5 and SD 2.1 models, you can use either + standard *safetensor* models or *diffusers* models +- For additional models, you can use *diffusers* models only +- You can download diffuser models directly from [Huggingface hub](https://huggingface.co/) + or use built-in model search & download in SD.Next: **UI -> Models -> Huggingface** +- Note that access to some models is gated + In which case, you need to accept model EULA and provide your huggingface token + +### Extra Networks + +- Lora networks +- Textual inversions (embeddings) + +Note that Lora and TI need are still model-specific, so you cannot use Lora trained on SD 1.5 on SD-XL +(just like you couldn't do it on SD 2.1 model) - it needs to be trained for a specific model + +Support for SD-XL training is expected shortly + +### Diffuser Settings + +- UI -> Settings -> Diffuser Settings + contains additional tunable parameters + +### Samplers + +- Samplers (schedulers) are pipeline specific, so when running with diffuser backend, you'll see a different list of samplers +- UI -> Settings -> Sampler Settings shows different configurable parameters depending on backend +- Recommended sampler for diffusers is **DEIS** + +### Other + +- Updated **System Info** tab with additional information +- Support for `lowvram` and `medvram` modes + Additional tunables are available in UI -> Settings -> Diffuser Settings +- Support for both default **SDP** and **xFormers** cross-optimizations + Other cross-optimization methods are not available +- **Extra Networks UI** will show available diffusers models +- **CUDA model compile** + UI Settings -> Compute settings + Requires GPU with high VRAM + Diffusers recommend `reduce overhead`, but other methods are available as well + Fullgraph is possible (with sufficient vram) when using diffusers + +## SD-XL Notes + +- SD-XL model is designed as two-stage model + You can run SD-XL pipeline using just `base` model, but for best results, load both `base` and `refiner` models + - `base`: Trained on images with variety of aspect ratios and uses OpenCLIP-ViT/G and CLIP-ViT/L for text encoding + - `refiner`: Trained to denoise small noise levels of high quality data and uses the OpenCLIP model +- If you want to use `refiner` model, it is advised to add `sd_model_refiner` to **quicksettings** + in UI Settings -> User Interface +- SD-XL model was trained on **1024px** images + You can use it with smaller sizes, but you will likely get better results with SD 1.5 models +- SD-XL model NSFW filter has been turned off + +## Limitations + +- Diffusers do not have callbacks per-step, so any functionality that relies on that will not be available + This includes trival but very visible **progress bar** +- Any extension that requires access to model internals will likely not work when using diffusers backend + This for example includes standard extensions such as `ControlNet`, `MultiDiffusion`, `LyCORIS` +- Second-pass workflows such as `hires fix` are not yet implemented (soon) +- Hypernetworks +- Explit VAE usage (soon) + +## Performance + +Comparison of original stable diffusion pipeline and diffusers pipeline + +| pipeline | performance it/s | memory cpu/gpu | +| --- | --- | --- | +| original | 7.99 / 7.93 / 8.83 / 9.14 / 9.2 | 6.7 / 7.2 | +| original medvram | 6.23 / 7.16 / 8.41 / 9.24 / 9.68 | 8.4 / 6.8 | +| original lowvram | 1.05 / 1.94 / 3.2 / 4.81 / 6.46 | 8.8 / 5.2 | +| diffusers | 9 / 7.4 / 8.2 / 8.4 / 7.0 | 4.3 / 9.0 | +| diffusers medvram | 7.5 / 6.7 / 7.5 / 7.8 / 7.2 | 6.6 / 8.2 | +| diffusers lowvram | 7.0 / 7.0 / 7.4 / 7.7 / 7.8 | 4.3 / 7.2 | +| diffusers with safetensors | 8.9 / 7.3 / 8.1 / 8.4 / 7.1 | 5.9 / 9.0 | + +Notes: + +- Performance is measured using standard SD 1.5 model +- Performance is measured for `batch-size` 1, 2, 4, 8 16 +- Test environment: + - nVidia RTX 3060 GPU + - Torch 2.1-nightly with CUDA 12.1 + - Cross-optimization: SDP +- All being equal, diffussers seem to: + - Use slightly less RAM and more VRAM + - Have highly efficient medvram/lowvram equivalents which don't loose a lot of performance + - Faster on smaller batch sizes, slower on larger batch sizes + +## TODO initial support merged into `dev` branch @@ -13,61 +138,6 @@ lora support is not compatible with setting `Use LyCoris handler for all Lora ty to update repo, do not use `--upgrade` flag, use manual `git pull` instead -## Test - -### Standard - -goal is to test standard workflows (so not diffusers) to ensure there are no regressions -so diffusers code can be merged into `master` and we can continue with development there - -- run with `webui --debug --backend original` - -### Diffusers - -whats implemented so far? - -- new scheduler: deis -- simple model downloader for huggingface models: tabs -> models -> hf hub -- use huggingface models -- extra networks ui -- use safetensor models with diffusers backend -- lowvram and medvram equivalents for diffusers -- standard workflows: - - txt2img, img2img, inpaint, outpaint, process - - hires fix, restore faces, etc? -- textual inversion - yes, this applies to standard embedddings, don't need ones from huggingface -- lora - yes, this applies to standard loras, don't need ones from huggingface - but seems that diffuser lora support is somewhat limited, so quite a few loras may not work - you should see which lora loads without issues in console log -- system info tab with updated information -- kandinsky model - works for me - -### Experimental - -- cuda model compile - in settings -> compute settings - diffusers recommend `reduce overhead`, but other methods are available as well - it seems that fullgraph is possible (with sufficient vram) when using diffusers -- deepfloyd - in theory it should work, but its 20gb model so cant test it just yet - note that access is gated, so you'll need to download using your huggingface credentials - (you can still do it from sdnext ui, just need access token) - -## Todo - -- sdxl model -- no idea if sd21 works out-of-the-box -- hires fix? -- vae support - -## Limitations - -even if extensions are not supported, runtime errors are never nice -will need to handle in the code before we get out of alpha - - lycoris `lyco_patch_lora` - controlnet @@ -76,28 +146,11 @@ will need to handle in the code before we get out of alpha > sd_model.first_stage_model?.encoder? - dynamic-thresholding > AttributeError: 'DiffusionSampler' object has no attribute 'model_wrap_cfg' -- no per-step callback -## Performance - -| pipeline | performance it/s | memory cpu/gpu | -| --- | --- | --- | -| original | 7.99 / 7.93 / 8.83 / 9.14 / 9.2 | 6.7 / 7.2 | -| original medvram | 6.23 / 7.16 / 8.41 / 9.24 / 9.68 | 8.4 / 6.8 | -| original lowvram | 1.05 / 1.94 / 3.2 / 4.81 / 6.46 | 8.8 / 5.2 | -| diffusers | 9 / 7.4 / 8.2 / 8.4 / 7.0 | 4.3 / 9.0 | -| diffusers medvram | 7.5 / 6.7 / 7.5 / 7.8 / 7.2 | 6.6 / 8.2 | -| diffusers lowvram | 7.0 / 7.0 / 7.4 / 7.7 / 7.8 | 4.3 / 7.2 | -| diffusers with safetensors | 8.9 / 7.3 / 8.1 / 8.4 / 7.1 | 5.9 / 9.0 | - -Notes: - -- Performance is measured for `batch-size` 1, 2, 4, 8 16 -- Test environment: - - nVidia RTX 3060 GPU - - Torch 2.1-nightly with CUDA 12.1 - - Cross-optimization: SDP -- All being equal, diffussers seem to: - - Use slightly less RAM and more VRAM - - Have highly efficient medvram/lowvram equivalents which don't loose a lot of performance - - Faster on smaller batch sizes, slower on larger batch sizes +- diffusers pipeline in general no sampler per-step callback, its completely opaque inside the pipeline + so i'm missing some very basic stuff like progress bar in the ui or ability to generate live preview based on intermediate latents +- StableDiffusionXLPipeline does not implement `from_ckpt` +- StableDiffusionXLPipeline has long delay after tqdm progress bar finishes and before it returns an image, i assume its vae, but its not a good user-experience +- VAE: + > vae = AutoencoderKL.from_pretrained("stabilityai/sdxl-vae") + > pipe = StableDiffusionPipeline.from_pretrained(model, vae=vae) diff --git a/installer.py b/installer.py index 0ebcc0680..d54e95eb8 100644 --- a/installer.py +++ b/installer.py @@ -283,6 +283,7 @@ def check_torch(): log.debug(f'Torch overrides: cuda={args.use_cuda} rocm={args.use_rocm} ipex={args.use_ipex} diml={args.use_directml}') log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml}') torch_command = os.environ.get('TORCH_COMMAND', '') + xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') if torch_command != '': pass elif allow_cuda and (shutil.which('nvidia-smi') is not None or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))): @@ -294,11 +295,9 @@ def check_torch(): os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', '10.3.0') os.environ.setdefault('PYTORCH_HIP_ALLOC_CONF', 'garbage_collection_threshold:0.8,max_split_size_mb:512') torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.1 torchvision==0.15.2 --index-url https://download.pytorch.org/whl/rocm5.4.2') - xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') elif allow_ipex and args.use_ipex and shutil.which('sycl-ls') is not None: log.info('Intel OneAPI Toolkit detected') torch_command = os.environ.get('TORCH_COMMAND', 'torch==1.13.0a0 torchvision==0.14.1a0 intel_extension_for_pytorch==1.13.120+xpu -f https://developer.intel.com/ipex-whl-stable-xpu') - xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') else: machine = platform.machine() if sys.platform == 'darwin': @@ -306,13 +305,11 @@ def check_torch(): elif allow_directml and args.use_directml and ('arm' not in machine and 'aarch' not in machine): log.info('Using DirectML Backend') torch_command = os.environ.get('TORCH_COMMAND', 'torch-directml') - xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') if 'torch' in torch_command and not args.version: install(torch_command, 'torch torchvision') else: log.info('Using CPU-only Torch') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') - xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') if 'torch' in torch_command and not args.version: install(torch_command, 'torch torchvision') if args.skip_torch: diff --git a/javascript/black-orange.css b/javascript/black-orange.css index 22c0e57c6..09cd5bb0f 100644 --- a/javascript/black-orange.css +++ b/javascript/black-orange.css @@ -1,10 +1,28 @@ /* generic html tags */ -:root { --font: "Source Sans Pro", 'ui-sans-serif', 'system-ui', "Roboto", sans-serif; } -html { font-size: 16px; } +:root { + --font: "Source Sans Pro", 'ui-sans-serif', 'system-ui', "Roboto", sans-serif; + --font-size: 16px; + --left-column: 490px; + --highlight-color: #ce6400; + --inactive-color: #4e1400; + --background-color: #000000; + --primary-50: #fff7ed; + --primary-100: #ffedd5; + --primary-200: #fed7aa; + --primary-300: #fdba74; + --primary-400: #fb923c; + --primary-500: #f97316; + --primary-600: #ea580c; + --primary-700: #c2410c; + --primary-800: #9a3412; + --primary-900: #7c2d12; + --primary-950: #6c2e12; +} +html { font-size: var(--font-size); } body, button, input, select, textarea { font-family: var(--font);} button { font-size: 1.2rem; } -img { background-color: black; } -input[type=range] { height: 18px; appearance: none; margin-top: 0; min-width: 160px; background-color: black; width: 100%; background: transparent; } +img { background-color: var(--background-color); } +input[type=range] { height: 18px; appearance: none; margin-top: 0; min-width: 160px; background-color: var(--background-color); width: 100%; background: transparent; } input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: #50555C; border-radius: 2px; border: 0px solid #222222; } input[type=range]::-moz-range-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: #50555C; border-radius: 2px; border: 0px solid #222222; } input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: 2px; background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; } @@ -14,10 +32,6 @@ input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111; border: 0 ::-webkit-scrollbar-thumb { background-color: var(--highlight-color); border-radius: 2px; border-width: 0; box-shadow: 2px 2px 3px #111111; } div.form { border-width: 0; box-shadow: none; background: transparent; overflow: visible; gap: 0.5em; margin-bottom: 6px; } -/* gradio shadowroot */ -.gradio-container { font-family: var(--font); --left-column: 490px; --highlight-color: #CE6400; --inactive-color: #4E1400; } - - /* gradio style classes */ fieldset .gr-block.gr-box, label.block span { padding: 0; margin-top: -4px; } .border-2 { border-width: 0; } @@ -27,11 +41,11 @@ fieldset .gr-block.gr-box, label.block span { padding: 0; margin-top: -4px; } .gr-button { font-weight: normal; box-shadow: 2px 2px 3px #111111; font-size: 0.8rem; min-width: 32px; min-height: 32px; padding: 3px; margin: 3px; } .gr-check-radio { background-color: var(--inactive-color); border-width: 0; border-radius: 2px; box-shadow: 2px 2px 3px #111111; } .gr-check-radio:checked { background-color: var(--highlight-color); } -.gr-compact { background-color: black; } +.gr-compact { background-color: var(--background-color); } .gr-form { border-width: 0; } .gr-input { background-color: #333333 !important; padding: 4px; margin: 4px; } .gr-input-label { color: lightyellow; border-width: 0; background: transparent; padding: 2px !important; } -.gr-panel { background-color: black; } +.gr-panel { background-color: var(--background-color); } .eta-bar { display: none !important } svg.feather.feather-image, .feather .feather-image { display: none } .gap-2 { padding-top: 8px; } @@ -42,9 +56,9 @@ svg.feather.feather-image, .feather .feather-image { display: none } .p-2 { padding: 0; } .px-4 { padding-lefT: 1rem; padding-right: 1rem; } .py-6 { padding-bottom: 0; } -.tabs { background-color: black; } +.tabs { background-color: var(--background-color); } .block.token-counter span { background-color: #222 !important; box-shadow: 2px 2px 2px #111; border: none !important; font-size: 0.8rem; } -.tab-nav { zoom: 120%; margin-bottom: 10px; border-bottom: 2px solid #CE6400 !important; padding-bottom: 2px; } +.tab-nav { zoom: 120%; margin-bottom: 10px; border-bottom: 2px solid var(--highlight-color) !important; padding-bottom: 2px; } .label-wrap { margin: 16px 0px 8px 0px; } .gradio-slider input[type="number"] { width: 4.5em; font-size: 0.8rem; } #tab_extensions table td, #tab_extensions table th { border: none; padding: 0.5em; } @@ -56,13 +70,13 @@ svg.feather.feather-image, .feather .feather-image { display: none } .progressDiv .progress { border-radius: 0 !important; background: var(--highlight-color); line-height: 3rem; height: 48px; } .gallery-item { box-shadow: none !important; } .performance { color: #888; } -.extra-networks { border-left: 2px solid #CE6400 !important; padding-left: 4px; } +.extra-networks { border-left: 2px solid var(--highlight-color) !important; padding-left: 4px; } .image-buttons { gap: 10px !important} /* gradio elements overrides */ #div.gradio-container { overflow-x: hidden; } #img2img_label_copy_to_img2img { font-weight: normal; } -#txt2img_prompt, #txt2img_neg_prompt, #img2img_prompt, #img2img_neg_prompt { background-color: black; box-shadow: 4px 4px 4px 0px #333333 !important; } +#txt2img_prompt, #txt2img_neg_prompt, #img2img_prompt, #img2img_neg_prompt { background-color: var(--background-color); box-shadow: 4px 4px 4px 0px #333333 !important; } #txt2img_prompt > label > textarea, #txt2img_neg_prompt > label > textarea, #img2img_prompt > label > textarea, #img2img_neg_prompt > label > textarea { font-size: 1.2rem; } #img2img_settings { min-width: calc(2 * var(--left-column)); max-width: calc(2 * var(--left-column)); background-color: #111111; padding-top: 16px; } #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; } @@ -82,7 +96,7 @@ svg.feather.feather-image, .feather .feather-image { display: none } #extras_upscale { margin-top: 10px } #txt2img_progress_row > div { min-width: var(--left-column); max-width: var(--left-column); } -#txt2img_results, #img2img_results, #extras_results { background-color: black; padding: 0; } +#txt2img_results, #img2img_results, #extras_results { background-color: var(--background-color); padding: 0; } #txt2img_seed_row { padding: 0; margin-top: 8px; } #txt2img_settings { min-width: var(--left-column); max-width: var(--left-column); background-color: #111111; padding-top: 16px; } #txt2img_subseed_row { padding: 0; margin-top: 16px; } @@ -97,13 +111,13 @@ svg.feather.feather-image, .feather .feather-image { display: none } /* based on gradio built-in dark theme */ :root, .light, .dark { - --body-background-fill: black; + --body-background-fill: var(--background-color); --body-text-color: var(--neutral-100); --color-accent-soft: var(--neutral-700); --background-fill-primary: #222222; --background-fill-secondary: none; - --border-color-accent: black; - --border-color-primary: black; + --border-color-accent: var(--background-color); + --border-color-primary: var(--background-color); --link-text-color-active: var(--secondary-500); --link-text-color: var(--secondary-500); --link-text-color-hover: var(--secondary-400); @@ -183,17 +197,6 @@ svg.feather.feather-image, .feather .feather-image { display: none } --button-secondary-border-color-hover: var(--button-secondary-border-color); --button-secondary-text-color: white; --button-secondary-text-color-hover: var(--button-secondary-text-color); - --primary-50: #fff7ed; - --primary-100: #ffedd5; - --primary-200: #fed7aa; - --primary-300: #fdba74; - --primary-400: #fb923c; - --primary-500: #f97316; - --primary-600: #ea580c; - --primary-700: #c2410c; - --primary-800: #9a3412; - --primary-900: #7c2d12; - --primary-950: #6c2e12; --secondary-50: #eff6ff; --secondary-100: #dbeafe; --secondary-200: #bfdbfe; diff --git a/modules/api/api.py b/modules/api/api.py index 144d5e82e..9664fc1a8 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -406,17 +406,15 @@ class Api: def interruptapi(self): shared.state.interrupt() - return {} def unloadapi(self): - unload_model_weights() - + unload_model_weights(op='model') + unload_model_weights(op='refiner') return {} def reloadapi(self): reload_model_weights() - return {} def skip(self): diff --git a/modules/processing.py b/modules/processing.py index f0e55acc2..6bf8b5eaf 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -223,7 +223,8 @@ class StableDiffusionProcessing: source_image = devices.cond_cast_float(source_image) # HACK: Using introspection as the Depth2Image model doesn't appear to uniquely # identify itself with a field common to all models. The conditioning_key is also hybrid. - if backend == Backend.DIFFUSERS: # TODO: Diffusers img2img_image_conditioning + if backend == Backend.DIFFUSERS: + log.warning('Diffusers not implemented: img2img_image_conditioning') return None if isinstance(self.sd_model, LatentDepth2ImageDiffusion): return self.depth2img_image_conditioning(source_image) @@ -682,11 +683,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: x_samples_ddim = torch.stack(x_samples_ddim).float() x_samples_ddim = torch.clamp((x_samples_ddim + 1.0) / 2.0, min=0.0, max=1.0) del samples_ddim - if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: - lowvram.send_everything_to_cpu() - devices.torch_gc() - if p.scripts is not None: - p.scripts.postprocess_batch(p, x_samples_ddim, batch_number=n) elif backend == Backend.DIFFUSERS: generator_device = 'cpu' if shared.opts.diffusers_generator_device == "cpu" else shared.device @@ -695,7 +691,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") - shared.sd_model.scheduler = sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op + # shared.sd_model.scheduler = sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op cross_attention_kwargs={} if lora_state['active']: @@ -708,23 +704,48 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING: # TODO(PVP): change out to latents once possible with `diffusers` task_specific_kwargs = {"image": p.init_images[0], "mask_image": p.image_mask, "strength": p.denoising_strength} - output = shared.sd_model( + + output = shared.sd_model( # pylint: disable=not-callable prompt=prompts, negative_prompt=negative_prompts, num_inference_steps=p.steps, guidance_scale=p.cfg_scale, generator=generator, - output_type="np", + output_type='np' if shared.sd_refiner is None else 'latent', cross_attention_kwargs=cross_attention_kwargs, **task_specific_kwargs ) + + if shared.sd_refiner is not None: + init_image = output.images[0] + output = shared.sd_refiner( # pylint: disable=not-callable + prompt=prompts, + negative_prompt=negative_prompts, + num_inference_steps=p.steps, + guidance_scale=p.cfg_scale, + generator=generator, + output_type='np', + cross_attention_kwargs=cross_attention_kwargs, + image=init_image + ) + x_samples_ddim = output.images + + if p.enable_hr: + log.warning('Diffusers not implemented: hires fix') + if lora_state['active']: unload_diffusers_lora() else: raise ValueError(f"Unknown backend {backend}") + if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: + lowvram.send_everything_to_cpu() + devices.torch_gc() + if p.scripts is not None: + p.scripts.postprocess_batch(p, x_samples_ddim, batch_number=n) + for i, x_sample in enumerate(x_samples_ddim): p.batch_index = i if backend == Backend.ORIGINAL: @@ -898,7 +919,23 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): if self.hr_upscaler is not None: self.extra_generation_params["Hires upscaler"] = self.hr_upscaler - def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): # TODO this is majority of processing time + def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): + + def save_intermediate(image, index): + """saves image before applying hires fix, if enabled in options; takes as an argument either an image or batch with latent space images""" + if not opts.save or self.do_not_save_samples or not opts.save_images_before_highres_fix: + return + if not isinstance(image, Image.Image): + image = sd_samplers.sample_to_image(image, index, approximation=0) + orig1 = self.extra_generation_params + orig2 = self.restore_faces + self.extra_generation_params = {} + self.restore_faces = False + info = create_infotext(self, self.all_prompts, self.all_seeds, self.all_subseeds, [], iteration=self.iteration, position_in_batch=index) + self.extra_generation_params = orig1 + self.restore_faces = orig2 + images.save_image(image, self.outpath_samples, "", seeds[index], prompts[index], opts.samples_format, info=info, suffix="-before-highres-fix") + if backend == Backend.DIFFUSERS: sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) @@ -916,21 +953,6 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): target_width = self.hr_upscale_to_x target_height = self.hr_upscale_to_y - def save_intermediate(image, index): - """saves image before applying hires fix, if enabled in options; takes as an argument either an image or batch with latent space images""" - if not opts.save or self.do_not_save_samples or not opts.save_images_before_highres_fix: - return - if not isinstance(image, Image.Image): - image = sd_samplers.sample_to_image(image, index, approximation=0) - orig1 = self.extra_generation_params - orig2 = self.restore_faces - self.extra_generation_params = {} - self.restore_faces = False - info = create_infotext(self, self.all_prompts, self.all_seeds, self.all_subseeds, [], iteration=self.iteration, position_in_batch=index) - self.extra_generation_params = orig1 - self.restore_faces = orig2 - images.save_image(image, self.outpath_samples, "", seeds[index], prompts[index], opts.samples_format, info=info, suffix="-before-highres-fix") - if latent_scale_mode is not None: for i in range(samples.shape[0]): save_intermediate(samples, i) diff --git a/modules/sd_models.py b/modules/sd_models.py index aad51403b..68e7a5a26 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -23,6 +23,7 @@ from modules.timer import Timer from modules.memstats import memory_stats from modules.paths_internal import models_path + transformers_logging.set_verbosity_error() model_dir = "Stable-diffusion" model_path = os.path.abspath(os.path.join(paths.models_path, model_dir)) @@ -127,7 +128,9 @@ def list_models(): checkpoints_list.clear() checkpoint_aliases.clear() ext_filter=[".safetensors"] if shared.opts.sd_disable_ckpt else [".ckpt", ".safetensors"] - model_list = modelloader.load_models(model_path=model_path, model_url=None, command_path=shared.opts.ckpt_dir, ext_filter=ext_filter, download_name=None, ext_blacklist=[".vae.ckpt", ".vae.safetensors"]) + model_list = [] + if shared.backend == shared.Backend.ORIGINAL or shared.opts.diffusers_pipeline == shared.pipelines[0]: + model_list += modelloader.load_models(model_path=model_path, model_url=None, command_path=shared.opts.ckpt_dir, ext_filter=ext_filter, download_name=None, ext_blacklist=[".vae.ckpt", ".vae.safetensors"]) if shared.backend == shared.Backend.DIFFUSERS: model_list += modelloader.load_diffusers_models(model_path=os.path.join(models_path, 'Diffusers'), command_path=shared.opts.diffusers_dir) @@ -210,11 +213,18 @@ def model_hash(filename): return 'NOHASH' -def select_checkpoint(model=True): - model_checkpoint = shared.opts.sd_model_checkpoint if model else shared.opts.sd_model_dict +def select_checkpoint(op='model'): + if op == 'model': + model_checkpoint = shared.opts.sd_model_checkpoint + elif op == 'dict': + model_checkpoint = shared.opts.sd_model_dict + elif op == 'refiner': + model_checkpoint = shared.opts.data['sd_model_refiner'] + if model_checkpoint is None or model_checkpoint == 'None': + return None checkpoint_info = get_closet_checkpoint_match(model_checkpoint) if checkpoint_info is not None: - shared.log.debug(f'Select checkpoint: {checkpoint_info.title if checkpoint_info is not None else None}') + shared.log.debug(f'Select checkpoint: {op} {checkpoint_info.title if checkpoint_info is not None else None}') return checkpoint_info if len(checkpoints_list) == 0: shared.log.error("Cannot run without a checkpoint") @@ -458,9 +468,10 @@ sd1_clip_weight = 'cond_stage_model.transformer.text_model.embeddings.token_embe sd2_clip_weight = 'cond_stage_model.model.transformer.resblocks.0.attn.in_proj_weight' -class SdModelData: +class ModelData: def __init__(self): self.sd_model = None + self.sd_refiner = None self.sd_dict = 'None' self.initial = True self.lock = threading.Lock() @@ -470,9 +481,9 @@ class SdModelData: with self.lock: try: if shared.backend == shared.Backend.ORIGINAL: - reload_model_weights() + reload_model_weights(op='model') elif shared.backend == shared.Backend.DIFFUSERS: - load_diffuser() + load_diffuser(op='model') else: shared.log.error(f"Unknown Stable Diffusion backend: {shared.backend}") self.initial = False @@ -483,11 +494,31 @@ class SdModelData: return self.sd_model def set_sd_model(self, v): + shared.log.debug(f"Class model: {v}") self.sd_model = v + def get_sd_refiner(self): + if self.sd_model is None: + with self.lock: + try: + if shared.backend == shared.Backend.ORIGINAL: + reload_model_weights(op='refiner') + elif shared.backend == shared.Backend.DIFFUSERS: + load_diffuser(op='refiner') + else: + shared.log.error(f"Unknown Stable Diffusion backend: {shared.backend}") + self.initial = False + except Exception as e: + shared.log.error("Failed to load stable diffusion model") + errors.display(e, "loading stable diffusion model") + self.sd_refiner = None + return self.sd_refiner -model_data = SdModelData() + def set_sd_refiner(self, v): + shared.log.debug(f"Class refiner: {v}") + self.sd_refiner = v +model_data = ModelData() class PriorPipeline: def __init__(self, prior, main): @@ -531,7 +562,7 @@ class PriorPipeline: return result -def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None): # pylint: disable=unused-argument +def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument if timer is None: timer = Timer() import logging @@ -548,27 +579,61 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if shared.opts.data['sd_model_checkpoint'] == 'model.ckpt': shared.opts.data['sd_model_checkpoint'] = "runwayml/stable-diffusion-v1-5" + + if op == 'model' or op == 'dict': + if model_data.sd_model is not None and (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model + return + else: + if model_data.sd_refiner is not None and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model + return + sd_model = None try: - devices.set_cuda_params() # todo + devices.set_cuda_params() if shared.cmd_opts.ckpt is not None and model_data.initial: # initial load model_name = modelloader.find_diffuser(shared.cmd_opts.ckpt) if model_name is not None: - shared.log.info(f'Loading diffuser model: {model_name}') + shared.log.info(f'Loading diffuser {op}: {model_name}') model_file = modelloader.download_diffusers_model(hub_id=model_name) sd_model = diffusers.DiffusionPipeline.from_pretrained(model_file, **diffusers_load_config) list_models() # rescan for downloaded model checkpoint_info = CheckpointInfo(model_name) if sd_model is None: - checkpoint_info = checkpoint_info or select_checkpoint() - shared.log.info(f'Loading diffuser model: {checkpoint_info.filename}') + checkpoint_info = checkpoint_info or select_checkpoint(op=op) + if checkpoint_info is None: + unload_model_weights(op=op) + return + shared.log.info(f'Loading diffuser {op}: {checkpoint_info.filename}') if not os.path.isfile(checkpoint_info.path): sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) else: diffusers_load_config["local_files_only "] = True diffusers_load_config["extract_ema"] = shared.opts.diffusers_extract_ema - sd_model = diffusers.StableDiffusionPipeline.from_ckpt(checkpoint_info.path, **diffusers_load_config) + try: + # pipelines = ['Stable Diffusion', 'Stable Diffusion XL', 'Kandinsky V1', 'Kandinsky V2', 'DeepFloyd IF', 'Shap-E'] + if shared.opts.diffusers_pipeline == shared.pipelines[0]: + pipeline = diffusers.StableDiffusionPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[1]: + pipeline = diffusers.StableDiffusionXLPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[2]: + pipeline = diffusers.KandinskyPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[3]: + pipeline = diffusers.KandinskyV22Pipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[4]: + pipeline = diffusers.IFPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[5]: + pipeline = diffusers.ShapEPipeline + else: + shared.log.error(f'Diffusers unknown pipeline: {shared.opts.diffusers_pipeline}') + except Exception as e: + shared.log.error(f'Diffusers failed initializing pipeline: {shared.opts.diffusers_pipeline} {e}') + return + try: + sd_model = pipeline.from_ckpt(checkpoint_info.path, **diffusers_load_config) + except Exception as e: + shared.log.error(f'Diffusers failed loading model using pipeline: {checkpoint_info.path} {shared.opts.diffusers_pipeline} {e}') + return if "StableDiffusion" in sd_model.__class__.__name__: pass # scheduler is created on first use @@ -615,7 +680,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.unet.to(memory_format=torch.channels_last) if shared.opts.cuda_compile and torch.cuda.is_available(): sd_model.to(devices.device) - import torch._dynamo # pylint: disable=unused-import + import torch._dynamo # pylint: disable=unused-import,redefined-outer-name log_level = logging.WARNING if shared.opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access torch._dynamo.config.verbose = shared.opts.cuda_compile_verbose # pylint: disable=protected-access @@ -632,7 +697,11 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No except Exception as e: shared.log.error("Failed to load diffusers model") errors.display(e, "loading Diffusers model") - shared.sd_model = sd_model + + if op == 'refiner': + model_data.sd_refiner = sd_model + else: + model_data.sd_model = sd_model from modules.textual_inversion import textual_inversion embedding_db = textual_inversion.EmbeddingDatabase() @@ -685,7 +754,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): new_pipe.sd_model_checkpoint = sd_model_checkpoint new_pipe.sd_model_hash = sd_model_hash - shared.sd_model = new_pipe + model_data.sd_model = new_pipe shared.log.info(f"Pipeline class changed from {pipe.__class__.__name__} to {new_pipe_cls.__name__}") @@ -700,21 +769,32 @@ def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType: return DiffusersTaskType.TEXT_2_IMAGE -def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None): +def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): from modules import lowvram, sd_hijack - checkpoint_info = checkpoint_info or select_checkpoint() + checkpoint_info = checkpoint_info or select_checkpoint(op=op) if checkpoint_info is None: return - if model_data.sd_model is not None and (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model - return - shared.log.debug(f'Load model: name={checkpoint_info.filename} dict={already_loaded_state_dict is not None}') + if op == 'model' or op == 'dict': + if model_data.sd_model is not None and (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model + return + else: + if model_data.sd_refiner is not None and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model + return + shared.log.debug(f'Load {op}: name={checkpoint_info.filename} dict={already_loaded_state_dict is not None}') if timer is None: timer = Timer() current_checkpoint_info = None - if model_data.sd_model is not None: - sd_hijack.model_hijack.undo_hijack(model_data.sd_model) - current_checkpoint_info = model_data.sd_model.sd_checkpoint_info - unload_model_weights() + if op == 'model' or op == 'dict': + if model_data.sd_model is not None: + sd_hijack.model_hijack.undo_hijack(model_data.sd_model) + current_checkpoint_info = model_data.sd_model.sd_checkpoint_info + unload_model_weights(op=op) + else: + if model_data.sd_refiner is not None: + sd_hijack.model_hijack.undo_hijack(model_data.sd_refiner) + current_checkpoint_info = model_data.sd_refiner.sd_checkpoint_info + unload_model_weights(op=op) + do_inpainting_hijack() devices.set_cuda_params() if already_loaded_state_dict is not None: @@ -760,7 +840,10 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None) sd_model = torch.xpu.optimize(sd_model, dtype=devices.dtype, auto_kernel_selection=True, optimize_lstm=True, graph_mode=True if shared.opts.cuda_compile and shared.opts.cuda_compile_mode == 'ipex' else False) shared.log.info("Applied IPEX Optimize") - model_data.sd_model = sd_model + if op == 'refiner': + model_data.sd_refiner = sd_model + else: + model_data.sd_model = sd_model sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings(force_reload=True) # Reload embeddings after model load as they may or may not fit the model timer.record("embeddings") script_callbacks.model_loaded_callback(sd_model) @@ -771,7 +854,7 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None) shared.log.info(f'Model load finished: {memory_stats()} cached={len(checkpoints_loaded.keys())}') -def reload_model_weights(sd_model=None, info=None, reuse_dict=False): +def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model'): load_dict = shared.opts.sd_model_dict != model_data.sd_dict global skip_next_load # pylint: disable=global-statement if skip_next_load: @@ -779,15 +862,18 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False): skip_next_load = False return from modules import lowvram, sd_hijack - checkpoint_info = info or select_checkpoint(model=not load_dict) # are we selecting model or dictionary - next_checkpoint_info = info or select_checkpoint(model=load_dict) if load_dict else None + checkpoint_info = info or select_checkpoint(op=op) # are we selecting model or dictionary + next_checkpoint_info = info or select_checkpoint(op='dict' if load_dict else 'model') if load_dict else None + if checkpoint_info is None: + unload_model_weights(op=op) + return if load_dict: shared.log.debug(f'Model dict: existing={sd_model is not None} target={checkpoint_info.filename} info={info}') else: model_data.sd_dict = 'None' shared.log.debug(f'Load model weights: existing={sd_model is not None} target={checkpoint_info.filename} info={info}') if not sd_model: - sd_model = model_data.sd_model + sd_model = model_data.sd_model if op == 'model' or op == 'dict' else model_data.sd_refiner if sd_model is None: # previous model load failed current_checkpoint_info = None else: @@ -802,7 +888,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False): shared.log.info('Reusing previous model dictionary') sd_hijack.model_hijack.undo_hijack(sd_model) else: - unload_model_weights() + unload_model_weights(op=op) sd_model = None timer = Timer() state_dict = get_checkpoint_state_dict(checkpoint_info, timer) @@ -811,14 +897,14 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False): if sd_model is None or checkpoint_config != sd_model.used_config: del sd_model if shared.backend == shared.Backend.ORIGINAL: - load_model(checkpoint_info, already_loaded_state_dict=state_dict, timer=timer) + load_model(checkpoint_info, already_loaded_state_dict=state_dict, timer=timer, op=op) else: - load_diffuser(checkpoint_info, already_loaded_state_dict=state_dict, timer=timer) + load_diffuser(checkpoint_info, already_loaded_state_dict=state_dict, timer=timer, op=op) if load_dict and next_checkpoint_info is not None: model_data.sd_dict = shared.opts.sd_model_dict shared.opts.data["sd_model_checkpoint"] = next_checkpoint_info.title reload_model_weights(reuse_dict=True) # ok we loaded dict now lets redo and load model on top of it - return model_data.sd_model + return model_data.sd_model if op == 'model' or op == 'dict' else model_data.sd_refiner try: load_model_weights(sd_model, checkpoint_info, state_dict, timer) except Exception: @@ -835,17 +921,22 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False): shared.log.info(f"Weights loaded in {timer.summary()}") -def unload_model_weights(sd_model=None, _info=None): +def unload_model_weights(op='model'): from modules import sd_hijack - if model_data.sd_model: - model_data.sd_model.to(devices.cpu) - if shared.backend == shared.Backend.ORIGINAL: - sd_hijack.model_hijack.undo_hijack(model_data.sd_model) - sd_model = None - model_data.sd_model = None - devices.torch_gc(force=True) - shared.log.debug(f'Model weights unloaded: {memory_stats()}') - return sd_model + if op == 'model' or op == 'dict': + if model_data.sd_model: + model_data.sd_model.to(devices.cpu) + if shared.backend == shared.Backend.ORIGINAL: + sd_hijack.model_hijack.undo_hijack(model_data.sd_model) + model_data.sd_model = None + else: + if model_data.sd_refiner: + model_data.sd_refiner.to(devices.cpu) + if shared.backend == shared.Backend.ORIGINAL: + sd_hijack.model_hijack.undo_hijack(model_data.sd_refiner) + model_data.sd_refiner = None + shared.log.debug(f'Weights unloaded {op}: {memory_stats()}') + devices.torch_gc(force=True) def apply_token_merging(sd_model, token_merging_ratio): diff --git a/modules/shared.py b/modules/shared.py index e02fadac3..b0514d278 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -38,6 +38,7 @@ hypernetworks = {} loaded_hypernetworks = [] gradio_theme = gr.themes.Base() settings_components = None +pipelines = ['Stable Diffusion', 'Stable Diffusion XL', 'Kandinsky V1', 'Kandinsky V2', 'DeepFloyd IF', 'Shap-E'] latent_upscale_default_mode = "Latent" latent_upscale_modes = { "Latent": {"mode": "bilinear", "antialias": False}, @@ -217,6 +218,10 @@ class OptionInfo: self.comment_after += f"({info})" return self + def html(self, info): + self.comment_after += f"{info}" + return self + def needs_restart(self): self.comment_after += " (requires restart)" return self @@ -295,8 +300,9 @@ else: # cuda cross_attention_optimization_default ="Scaled-Dot-Product" options_templates.update(options_section(('sd', "Stable Diffusion"), { - "sd_model_checkpoint": OptionInfo(default_checkpoint, "Stable Diffusion checkpoint", gr.Dropdown, lambda: {"choices": list_checkpoint_tiles()}, refresh=refresh_checkpoints), "sd_checkpoint_autoload": OptionInfo(True, "Stable Diffusion checkpoint autoload on server start"), + "sd_model_checkpoint": OptionInfo(default_checkpoint, "Stable Diffusion checkpoint", gr.Dropdown, lambda: {"choices": list_checkpoint_tiles()}, refresh=refresh_checkpoints), + "sd_model_refiner": OptionInfo('None', "Stable Diffusion refiner", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints), "sd_checkpoint_cache": OptionInfo(0, "Number of cached model checkpoints", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}), "sd_vae_checkpoint_cache": OptionInfo(0, "Number of cached VAE checkpoints", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}), "sd_vae": OptionInfo("Automatic", "Select VAE", gr.Dropdown, lambda: {"choices": shared_items.sd_vae_items()}, refresh=shared_items.refresh_vae_list), @@ -341,10 +347,11 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "cuda_compile_fullgraph": OptionInfo(False, "Model compile fullgraph"), "cuda_compile_verbose": OptionInfo(False, "Model compile verbose mode"), "cuda_compile_errors": OptionInfo(True, "Model compile suppress errors"), - "disable_gc": OptionInfo(False, "Disable Torch memory garbage collection"), + "disable_gc": OptionInfo(True, "Disable Torch memory garbage collection on each generation"), })) options_templates.update(options_section(('diffusers', "Diffusers Settings"), { + "diffusers_pipeline": OptionInfo(pipelines[0], 'Diffuser Pipeline', gr.Dropdown, lambda: {"choices": pipelines}), "diffusers_extract_ema": OptionInfo(True, "Use model EMA weights when possible"), "diffusers_generator_device": OptionInfo("default", "Generator device", gr.Radio, lambda: {"choices": ["default", "cpu"]}), "diffusers_seq_cpu_offload": OptionInfo(False, "Enable sequential CPU offload"), @@ -905,7 +912,6 @@ class Shared(sys.modules[__name__].__class__): @property def sd_model(self): import modules.sd_models # pylint: disable=W0621 - # return modules.sd_models.model_data.sd_model return modules.sd_models.model_data.get_sd_model() @sd_model.setter @@ -913,6 +919,17 @@ class Shared(sys.modules[__name__].__class__): import modules.sd_models # pylint: disable=W0621 modules.sd_models.model_data.set_sd_model(value) -# sd_model: LatentDiffusion = None # this var is here just for IDE's type checking; it cannot be accessed because the class field above will be accessed instead + @property + def sd_refiner(self): + import modules.sd_models # pylint: disable=W0621 + return modules.sd_models.model_data.get_sd_refiner() + + @sd_refiner.setter + def sd_refiner(self, value): + import modules.sd_models # pylint: disable=W0621 + modules.sd_models.model_data.set_sd_refiner(value) + + sd_model = None +sd_refiner = None sys.modules[__name__].__class__ = Shared diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index 27170eb40..645eacdc9 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -145,8 +145,12 @@ class EmbeddingDatabase: self.word_embeddings[name] = embedding except Exception: self.skipped_embeddings[name] = embedding - text_inv_tokens = pipe.tokenizer.added_tokens_encoder.keys() - text_inv_tokens = [t for t in text_inv_tokens if not (len(t.split("_")) > 1 and t.split("_")[-1].isdigit())] + try: + text_inv_tokens = pipe.tokenizer.added_tokens_encoder.keys() + text_inv_tokens = [t for t in text_inv_tokens if not (len(t.split("_")) > 1 and t.split("_")[-1].isdigit())] + except Exception: + text_inv_tokens = [] + pass def load_from_file(self, path, filename): name, ext = os.path.splitext(filename) diff --git a/modules/ui.py b/modules/ui.py index 60ccc6cd1..37b6c0363 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -1031,7 +1031,8 @@ def create_ui(startup_timer = None): create_dirty_indicator("show_all_pages", [], interactive=False) def unload_sd_weights(): - modules.sd_models.unload_model_weights() + modules.sd_models.unload_model_weights(op='model') + modules.sd_models.unload_model_weights(op='refiner') def reload_sd_weights(): modules.sd_models.reload_model_weights() diff --git a/webui.py b/webui.py index b78919c47..930e4cd3a 100644 --- a/webui.py +++ b/webui.py @@ -167,14 +167,18 @@ def load_model(): if opts.sd_checkpoint_autoload: shared.state.begin() shared.state.job = 'load model' - thread = Thread(target=lambda: shared.sd_model) - thread.start() + thread_model = Thread(target=lambda: shared.sd_model) + thread_model.start() + thread_refiner = Thread(target=lambda: shared.sd_refiner) + thread_refiner.start() shared.state.end() - thread.join() + thread_model.join() + thread_refiner.join() else: log.debug('Model auto load disabled') - shared.opts.onchange("sd_model_checkpoint", wrap_queued_call(lambda: modules.sd_models.reload_model_weights()), call=False) - shared.opts.onchange("sd_model_dict", wrap_queued_call(lambda: modules.sd_models.reload_model_weights()), call=False) + shared.opts.onchange("sd_model_checkpoint", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='model')), call=False) + shared.opts.onchange("sd_model_refiner", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='refiner')), call=False) + shared.opts.onchange("sd_model_dict", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='dict')), call=False) startup_timer.record("checkpoint") diff --git a/wiki b/wiki index f941746c0..28e3cc15e 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit f941746c0ed2afcaa37c1ec77b86da4dee131bee +Subproject commit 28e3cc15ef4566564764fa73542ab6b00d2b0959 From 9f96d4f657178ddabe495d3a59916d57080505ac Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jul 2023 20:21:01 -0400 Subject: [PATCH 27/42] update notes --- DIFFUSERS.md | 164 +++++-------------------------------------- modules/sd_models.py | 15 +++- modules/shared.py | 9 ++- webui.py | 1 + 4 files changed, 39 insertions(+), 150 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index 61ff1c17b..1171920e1 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -1,156 +1,30 @@ -# Additional Models +# Diffusers -SD.Next includes *experimental* support for additional model pipelines -This includes support for additional models such as: - -- **Stable Diffusion XL** -- **Kandinsky** -- **Deep Floyd IF** -- **Shap-E** - -Note that support is *experimental*, do not open [GitHub issues](https://github.com/vladmandic/automatic/issues) for those models -and instead reach-out on [Discord](https://discord.gg/WqMzTUDC) using dedicated channels - -*This has been made possible by integration of [huggingface diffusers](https://huggingface.co/docs/diffusers/index) library with help of huggingface team!* - -## How to - -- Install **SD.Next** as usual -- Start with - `webui --backend diffusers` -- To go back to standard execution pipeline, start with - `webui --backend original` - -## Integration - -### Standard workflows - -- **txt2txt** -- **img2img** -- **process** - -### Model Access - -- For standard SD 1.5 and SD 2.1 models, you can use either - standard *safetensor* models or *diffusers* models -- For additional models, you can use *diffusers* models only -- You can download diffuser models directly from [Huggingface hub](https://huggingface.co/) - or use built-in model search & download in SD.Next: **UI -> Models -> Huggingface** -- Note that access to some models is gated - In which case, you need to accept model EULA and provide your huggingface token - -### Extra Networks - -- Lora networks -- Textual inversions (embeddings) - -Note that Lora and TI need are still model-specific, so you cannot use Lora trained on SD 1.5 on SD-XL -(just like you couldn't do it on SD 2.1 model) - it needs to be trained for a specific model - -Support for SD-XL training is expected shortly - -### Diffuser Settings - -- UI -> Settings -> Diffuser Settings - contains additional tunable parameters - -### Samplers - -- Samplers (schedulers) are pipeline specific, so when running with diffuser backend, you'll see a different list of samplers -- UI -> Settings -> Sampler Settings shows different configurable parameters depending on backend -- Recommended sampler for diffusers is **DEIS** - -### Other - -- Updated **System Info** tab with additional information -- Support for `lowvram` and `medvram` modes - Additional tunables are available in UI -> Settings -> Diffuser Settings -- Support for both default **SDP** and **xFormers** cross-optimizations - Other cross-optimization methods are not available -- **Extra Networks UI** will show available diffusers models -- **CUDA model compile** - UI Settings -> Compute settings - Requires GPU with high VRAM - Diffusers recommend `reduce overhead`, but other methods are available as well - Fullgraph is possible (with sufficient vram) when using diffusers - -## SD-XL Notes - -- SD-XL model is designed as two-stage model - You can run SD-XL pipeline using just `base` model, but for best results, load both `base` and `refiner` models - - `base`: Trained on images with variety of aspect ratios and uses OpenCLIP-ViT/G and CLIP-ViT/L for text encoding - - `refiner`: Trained to denoise small noise levels of high quality data and uses the OpenCLIP model -- If you want to use `refiner` model, it is advised to add `sd_model_refiner` to **quicksettings** - in UI Settings -> User Interface -- SD-XL model was trained on **1024px** images - You can use it with smaller sizes, but you will likely get better results with SD 1.5 models -- SD-XL model NSFW filter has been turned off - -## Limitations - -- Diffusers do not have callbacks per-step, so any functionality that relies on that will not be available - This includes trival but very visible **progress bar** -- Any extension that requires access to model internals will likely not work when using diffusers backend - This for example includes standard extensions such as `ControlNet`, `MultiDiffusion`, `LyCORIS` -- Second-pass workflows such as `hires fix` are not yet implemented (soon) -- Hypernetworks -- Explit VAE usage (soon) - -## Performance - -Comparison of original stable diffusion pipeline and diffusers pipeline - -| pipeline | performance it/s | memory cpu/gpu | -| --- | --- | --- | -| original | 7.99 / 7.93 / 8.83 / 9.14 / 9.2 | 6.7 / 7.2 | -| original medvram | 6.23 / 7.16 / 8.41 / 9.24 / 9.68 | 8.4 / 6.8 | -| original lowvram | 1.05 / 1.94 / 3.2 / 4.81 / 6.46 | 8.8 / 5.2 | -| diffusers | 9 / 7.4 / 8.2 / 8.4 / 7.0 | 4.3 / 9.0 | -| diffusers medvram | 7.5 / 6.7 / 7.5 / 7.8 / 7.2 | 6.6 / 8.2 | -| diffusers lowvram | 7.0 / 7.0 / 7.4 / 7.7 / 7.8 | 4.3 / 7.2 | -| diffusers with safetensors | 8.9 / 7.3 / 8.1 / 8.4 / 7.1 | 5.9 / 9.0 | - -Notes: - -- Performance is measured using standard SD 1.5 model -- Performance is measured for `batch-size` 1, 2, 4, 8 16 -- Test environment: - - nVidia RTX 3060 GPU - - Torch 2.1-nightly with CUDA 12.1 - - Cross-optimization: SDP -- All being equal, diffussers seem to: - - Use slightly less RAM and more VRAM - - Have highly efficient medvram/lowvram equivalents which don't loose a lot of performance - - Faster on smaller batch sizes, slower on larger batch sizes - -## TODO +## Install initial support merged into `dev` branch - git clone https://github.com/vladmandic/automatic -b dev diffusers - cd diffusers - webui --debug --backend diffusers +- first download and start as normal: + > git clone https://github.com/vladmandic/automatic -b dev diffusers + > cd diffusers + > webui --debug --backend original -default sd 1.5 model will be downloaded automatically to `models/Diffusers` +- then upgrade diffusers to unreleased version and switch to using diffusers + > pip install --upgrade git+https://github.com/huggingface/diffusers + > webui --debug --quick --backend diffusers -on first startup, disable **controlnet** and **multi-diffusion** extensions as right now they are not compatible with diffusers -lora support is not compatible with setting `Use LyCoris handler for all Lora types`, make sure its disabled +- to go back to standard execution pipeline, start with + > webui --debug --backend original -to update repo, do not use `--upgrade` flag, use manual `git pull` instead +- To update repo, do not use `--upgrade` flag, use manual `git pull` instead -- lycoris - `lyco_patch_lora` -- controlnet - > sd_model.model?.diffusion_model? -- multi-diffusion - > sd_model.first_stage_model?.encoder? -- dynamic-thresholding - > AttributeError: 'DiffusionSampler' object has no attribute 'model_wrap_cfg' +## Notes -- diffusers pipeline in general no sampler per-step callback, its completely opaque inside the pipeline - so i'm missing some very basic stuff like progress bar in the ui or ability to generate live preview based on intermediate latents -- StableDiffusionXLPipeline does not implement `from_ckpt` -- StableDiffusionXLPipeline has long delay after tqdm progress bar finishes and before it returns an image, i assume its vae, but its not a good user-experience -- VAE: +All notes have moved to [Wiki page](https://github.com/vladmandic/automatic/wiki/Diffusers) + +## TODO + +- VAE > vae = AutoencoderKL.from_pretrained("stabilityai/sdxl-vae") > pipe = StableDiffusionPipeline.from_pretrained(model, vae=vae) +- Refiner handler with medvram/lowvram diff --git a/modules/sd_models.py b/modules/sd_models.py index 68e7a5a26..83661836c 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -219,7 +219,7 @@ def select_checkpoint(op='model'): elif op == 'dict': model_checkpoint = shared.opts.sd_model_dict elif op == 'refiner': - model_checkpoint = shared.opts.data['sd_model_refiner'] + model_checkpoint = shared.opts.data.get('sd_model_refiner', None) if model_checkpoint is None or model_checkpoint == 'None': return None checkpoint_info = get_closet_checkpoint_match(model_checkpoint) @@ -595,7 +595,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if model_name is not None: shared.log.info(f'Loading diffuser {op}: {model_name}') model_file = modelloader.download_diffusers_model(hub_id=model_name) - sd_model = diffusers.DiffusionPipeline.from_pretrained(model_file, **diffusers_load_config) + try: + sd_model = diffusers.DiffusionPipeline.from_pretrained(model_file, **diffusers_load_config) + except Exception as e: + shared.log.error(f'Diffusers failed loading model: {model_file} {e}') list_models() # rescan for downloaded model checkpoint_info = CheckpointInfo(model_name) @@ -606,7 +609,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No return shared.log.info(f'Loading diffuser {op}: {checkpoint_info.filename}') if not os.path.isfile(checkpoint_info.path): - sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) + try: + sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) + except Exception as e: + shared.log.error(f'Diffusers failed loading model: {checkpoint_info.path} {e}') else: diffusers_load_config["local_files_only "] = True diffusers_load_config["extract_ema"] = shared.opts.diffusers_extract_ema @@ -690,6 +696,9 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model("dummy prompt") shared.log.info("Complilation done.") + if sd_model is None: + shared.log.error('Diffuser model not loaded') + return sd_model.sd_checkpoint_info = checkpoint_info # pylint: disable=attribute-defined-outside-init sd_model.sd_model_checkpoint = checkpoint_info.filename # pylint: disable=attribute-defined-outside-init sd_model.sd_model_hash = checkpoint_info.hash # pylint: disable=attribute-defined-outside-init diff --git a/modules/shared.py b/modules/shared.py index b0514d278..a0aa8be8d 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -264,12 +264,17 @@ def list_themes(): return themes -def lora_disable(): +def disable_extensions(): if opts.lora_disable: if 'Lora' not in opts.disabled_extensions: opts.data['disabled_extensions'].append('Lora') else: opts.data['disabled_extensions'] = [x for x in opts.disabled_extensions if x != 'Lora'] + if backend == Backend.DIFFUSERS: + for ext in ['sd-webui-controlnet', 'sd-dynamic-thresholding', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris']: + if ext not in opts.disabled_extensions: + log.warning(f'Diffusers disabling uncompatible extension: {ext}') + opts.data['disabled_extensions'].append(ext) def refresh_themes(): @@ -586,7 +591,7 @@ options_templates.update(options_section(('extra_networks', "Extra Networks"), { "extra_networks_card_fit": OptionInfo("cover", "UI image contain method", gr.Radio, lambda: {"choices": ["contain", "cover", "fill"]}), "extra_network_skip_indexing": OptionInfo(False, "Do not automatically build extra network pages", gr.Checkbox), "lyco_patch_lora": OptionInfo(False, "Use LyCoris handler for all Lora types", gr.Checkbox), - "lora_disable": OptionInfo(False, "Disable built-in Lora handler", gr.Checkbox, { "visible": True }, onchange=lora_disable), + "lora_disable": OptionInfo(False, "Disable built-in Lora handler", gr.Checkbox, { "visible": True }, onchange=disable_extensions), "lora_functional": OptionInfo(False, "Use Kohya method for handling multiple Loras", gr.Checkbox), "extra_networks_add_text_separator": OptionInfo(" ", "Extra text to add before <...> when adding extra network to prompt", gr.Text, { "visible": False }), "extra_networks_default_multiplier": OptionInfo(1.0, "Multiplier for extra networks", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), diff --git a/webui.py b/webui.py index 930e4cd3a..1b8da00c7 100644 --- a/webui.py +++ b/webui.py @@ -101,6 +101,7 @@ def check_rollback_vae(): def initialize(): log.debug('Entering initialize') + shared.disable_extensions() check_rollback_vae() modules.sd_vae.refresh_vae_list() From 0157b3bb4caac4dff89c7c179b1ddf4dbcfccabe Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jul 2023 20:24:04 -0400 Subject: [PATCH 28/42] update --- DIFFUSERS.md | 6 +----- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index 1171920e1..47b86af47 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -4,15 +4,11 @@ initial support merged into `dev` branch -- first download and start as normal: +- download from branch and start as normal: > git clone https://github.com/vladmandic/automatic -b dev diffusers > cd diffusers > webui --debug --backend original -- then upgrade diffusers to unreleased version and switch to using diffusers - > pip install --upgrade git+https://github.com/huggingface/diffusers - > webui --debug --quick --backend diffusers - - to go back to standard execution pipeline, start with > webui --debug --backend original diff --git a/requirements.txt b/requirements.txt index cdd498173..6dc79e46a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,7 +49,7 @@ requests==2.31.0 tqdm==4.65.0 accelerate==0.20.3 opencv-python==4.7.0.72 -diffusers==0.17.1 +diffusers==0.18.0 einops==0.4.1 gradio==3.32.0 numexpr==2.8.4 From 1d36e19996d7ceafc5283e54eda02c5ae9dd403b Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Fri, 7 Jul 2023 18:52:04 +0900 Subject: [PATCH 29/42] Fix PNDMScheduler for DirectML. --- modules/dml/hijack/__init__.py | 1 + modules/dml/hijack/diffusers.py | 47 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 modules/dml/hijack/diffusers.py diff --git a/modules/dml/hijack/__init__.py b/modules/dml/hijack/__init__.py index d8cc0913a..7e3424775 100644 --- a/modules/dml/hijack/__init__.py +++ b/modules/dml/hijack/__init__.py @@ -3,3 +3,4 @@ import modules.dml.hijack.stablediffusion import modules.dml.hijack.torch import modules.dml.hijack.realesrgan_model import modules.dml.hijack.plms +import modules.dml.hijack.diffusers diff --git a/modules/dml/hijack/diffusers.py b/modules/dml/hijack/diffusers.py new file mode 100644 index 000000000..42888fb8c --- /dev/null +++ b/modules/dml/hijack/diffusers.py @@ -0,0 +1,47 @@ +import diffusers + +def _get_prev_sample(self, sample, timestep, prev_timestep, model_output): + # See formula (9) of PNDM paper https://arxiv.org/pdf/2202.09778.pdf + # this function computes x_(t−δ) using the formula of (9) + # Note that x_t needs to be added to both sides of the equation + + # Notation ( -> + # alpha_prod_t -> α_t + # alpha_prod_t_prev -> α_(t−δ) + # beta_prod_t -> (1 - α_t) + # beta_prod_t_prev -> (1 - α_(t−δ)) + # sample -> x_t + # model_output -> e_θ(x_t, t) + # prev_sample -> x_(t−δ) + print(sample) # DML Solution: PNDM Sampling does not work without this print. (because it depends on PLMS) + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + if self.config.prediction_type == "v_prediction": + model_output = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + elif self.config.prediction_type != "epsilon": + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon` or `v_prediction`" + ) + + # corresponds to (α_(t−δ) - α_t) divided by + # denominator of x_t in formula (9) and plus 1 + # Note: (α_(t−δ) - α_t) / (sqrt(α_t) * (sqrt(α_(t−δ)) + sqr(α_t))) = + # sqrt(α_(t−δ)) / sqrt(α_t)) + sample_coeff = (alpha_prod_t_prev / alpha_prod_t) ** (0.5) + + # corresponds to denominator of e_θ(x_t, t) in formula (9) + model_output_denom_coeff = alpha_prod_t * beta_prod_t_prev ** (0.5) + ( + alpha_prod_t * beta_prod_t * alpha_prod_t_prev + ) ** (0.5) + + # full formula (9) + prev_sample = ( + sample_coeff * sample - (alpha_prod_t_prev - alpha_prod_t) * model_output / model_output_denom_coeff + ) + + return prev_sample + +diffusers.PNDMScheduler._get_prev_sample = _get_prev_sample From 47c96e34d4ae6e7713f423607565278c351595ec Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Fri, 7 Jul 2023 19:11:18 +0900 Subject: [PATCH 30/42] Stringify tensor instead of printing. --- modules/dml/hijack/diffusers.py | 5 +++-- modules/dml/hijack/plms.py | 2 +- modules/dml/hijack/stablediffusion.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/dml/hijack/diffusers.py b/modules/dml/hijack/diffusers.py index 42888fb8c..3279398cf 100644 --- a/modules/dml/hijack/diffusers.py +++ b/modules/dml/hijack/diffusers.py @@ -1,6 +1,7 @@ +from torch import FloatTensor import diffusers -def _get_prev_sample(self, sample, timestep, prev_timestep, model_output): +def _get_prev_sample(self, sample: FloatTensor, timestep, prev_timestep, model_output): # See formula (9) of PNDM paper https://arxiv.org/pdf/2202.09778.pdf # this function computes x_(t−δ) using the formula of (9) # Note that x_t needs to be added to both sides of the equation @@ -13,7 +14,7 @@ def _get_prev_sample(self, sample, timestep, prev_timestep, model_output): # sample -> x_t # model_output -> e_θ(x_t, t) # prev_sample -> x_(t−δ) - print(sample) # DML Solution: PNDM Sampling does not work without this print. (because it depends on PLMS) + sample.__str__() # DML Solution: PNDM Sampling does not work without 'stringify'. (because it depends on PLMS) alpha_prod_t = self.alphas_cumprod[timestep] alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod beta_prod_t = 1 - alpha_prod_t diff --git a/modules/dml/hijack/plms.py b/modules/dml/hijack/plms.py index 5a9570454..2baef815d 100644 --- a/modules/dml/hijack/plms.py +++ b/modules/dml/hijack/plms.py @@ -48,7 +48,7 @@ def p_sample_plms(self, x, c, t, index, repeat_noise=False, use_original_steps=F def get_x_prev_and_pred_x0(e_t, index): # select parameters corresponding to the currently considered timestep - print(alphas[index]) # DML Solution: PLMS Sampling does not work without this print. + alphas[index].__str__() # DML Solution: PLMS Sampling does not work without this 'stringify'. a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) diff --git a/modules/dml/hijack/stablediffusion.py b/modules/dml/hijack/stablediffusion.py index cbc4b85fb..b14b0ece1 100644 --- a/modules/dml/hijack/stablediffusion.py +++ b/modules/dml/hijack/stablediffusion.py @@ -51,7 +51,7 @@ def p_sample_ddim(self, x, c, t, index, repeat_noise=False, use_original_steps=F sqrt_one_minus_alphas = self.model.sqrt_one_minus_alphas_cumprod if use_original_steps else self.ddim_sqrt_one_minus_alphas sigmas = self.model.ddim_sigmas_for_original_num_steps if use_original_steps else self.ddim_sigmas # select parameters corresponding to the currently considered timestep - print(alphas[index]) # DML Solution: DDIM Sampling does not work without this print. + alphas[index].__str__() # DML Solution: DDIM Sampling does not work without this 'stringify'. a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) From 3e1a6a96d07933c1397dec6946e834ab35720da1 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 7 Jul 2023 09:38:16 -0400 Subject: [PATCH 31/42] add additional pipelines --- CHANGELOG.md | 4 +++- DIFFUSERS.md | 2 +- modules/processing.py | 12 +++++++++++ modules/sd_models.py | 35 +++++++++++++++++++++++++++----- modules/sd_vae.py | 39 ++++++++++++++++++++++++++++++------ modules/shared.py | 8 ++++++-- modules/ui_extra_networks.py | 14 +++++++++---- 7 files changed, 95 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c4447625..18d85957b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,10 @@ - add settings -> extra networks -> do not automatically build extra network pages speeds up app start if you have a lot of extra networks and you want to build them manually when needed - extra network ui tweaks +- cache extra networks between tabs + this should result in neat 2x speedup on building extra networks - merge experimental diffusers support - this will be covered in details in separate post + covered in details in a separate post ## Update for 07/01/2023 diff --git a/DIFFUSERS.md b/DIFFUSERS.md index 47b86af47..df2c462fc 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -7,7 +7,7 @@ initial support merged into `dev` branch - download from branch and start as normal: > git clone https://github.com/vladmandic/automatic -b dev diffusers > cd diffusers - > webui --debug --backend original + > webui --debug --backend diffusers - to go back to standard execution pipeline, start with > webui --debug --backend original diff --git a/modules/processing.py b/modules/processing.py index 6bf8b5eaf..04058d519 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -705,12 +705,22 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: # TODO(PVP): change out to latents once possible with `diffusers` task_specific_kwargs = {"image": p.init_images[0], "mask_image": p.image_mask, "strength": p.denoising_strength} + def diffusers_callback(step: int, _timestep: int, latents: torch.FloatTensor): # TODO simplified callback for now + shared.state.sampling_step = step + shared.state.sampling_steps = p.steps + shared.state.current_latent = latents + shared.state.set_current_image() + if p.scripts is not None: + p.scripts.process(p) + output = shared.sd_model( # pylint: disable=not-callable prompt=prompts, negative_prompt=negative_prompts, num_inference_steps=p.steps, guidance_scale=p.cfg_scale, generator=generator, + callback_steps = 1, + callback = diffusers_callback, output_type='np' if shared.sd_refiner is None else 'latent', cross_attention_kwargs=cross_attention_kwargs, **task_specific_kwargs @@ -724,6 +734,8 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: num_inference_steps=p.steps, guidance_scale=p.cfg_scale, generator=generator, + callback_steps = 1, + callback = diffusers_callback, output_type='np', cross_attention_kwargs=cross_attention_kwargs, image=init_image diff --git a/modules/sd_models.py b/modules/sd_models.py index 83661836c..688aa3638 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -129,7 +129,7 @@ def list_models(): checkpoint_aliases.clear() ext_filter=[".safetensors"] if shared.opts.sd_disable_ckpt else [".ckpt", ".safetensors"] model_list = [] - if shared.backend == shared.Backend.ORIGINAL or shared.opts.diffusers_pipeline == shared.pipelines[0]: + if shared.backend == shared.Backend.ORIGINAL or shared.opts.diffusers_allow_safetensors: model_list += modelloader.load_models(model_path=model_path, model_url=None, command_path=shared.opts.ckpt_dir, ext_filter=ext_filter, download_name=None, ext_blacklist=[".vae.ckpt", ".vae.safetensors"]) if shared.backend == shared.Backend.DIFFUSERS: model_list += modelloader.load_diffusers_models(model_path=os.path.join(models_path, 'Diffusers'), command_path=shared.opts.diffusers_dir) @@ -577,7 +577,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No # "use_safetensors": True, # TODO(PVP) - we can't enable this for all checkpoints just yet } - if shared.opts.data['sd_model_checkpoint'] == 'model.ckpt': + if shared.opts.data.get('sd_model_checkpoint', '') == 'model.ckpt' or shared.opts.data.get('sd_model_checkpoint', '') == '': shared.opts.data['sd_model_checkpoint'] = "runwayml/stable-diffusion-v1-5" if op == 'model' or op == 'dict': @@ -608,6 +608,12 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No unload_model_weights(op=op) return shared.log.info(f'Loading diffuser {op}: {checkpoint_info.filename}') + + vae_file, vae_source = sd_vae.resolve_vae(checkpoint_info.filename) + vae = sd_vae.load_vae_diffusers(None, vae_file, vae_source) + if vae is not None: + diffusers_load_config["vae"] = vae + if not os.path.isfile(checkpoint_info.path): try: sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) @@ -617,7 +623,6 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No diffusers_load_config["local_files_only "] = True diffusers_load_config["extract_ema"] = shared.opts.diffusers_extract_ema try: - # pipelines = ['Stable Diffusion', 'Stable Diffusion XL', 'Kandinsky V1', 'Kandinsky V2', 'DeepFloyd IF', 'Shap-E'] if shared.opts.diffusers_pipeline == shared.pipelines[0]: pipeline = diffusers.StableDiffusionPipeline elif shared.opts.diffusers_pipeline == shared.pipelines[1]: @@ -630,13 +635,32 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No pipeline = diffusers.IFPipeline elif shared.opts.diffusers_pipeline == shared.pipelines[5]: pipeline = diffusers.ShapEPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[6]: + pipeline = diffusers.StableDiffusionImg2ImgPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[7]: + pipeline = diffusers.StableDiffusionXLImg2ImgPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[8]: + pipeline = diffusers.KandinskyImg2ImgPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[9]: + pipeline = diffusers.KandinskyV22Img2ImgPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[10]: + pipeline = diffusers.IFImg2ImgPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[11]: + pipeline = diffusers.ShapEImg2ImgPipeline else: shared.log.error(f'Diffusers unknown pipeline: {shared.opts.diffusers_pipeline}') except Exception as e: shared.log.error(f'Diffusers failed initializing pipeline: {shared.opts.diffusers_pipeline} {e}') return try: - sd_model = pipeline.from_ckpt(checkpoint_info.path, **diffusers_load_config) + if hasattr(pipeline, 'from_single_file'): + diffusers_load_config['use_safetensors'] = True + sd_model = pipeline.from_single_file(checkpoint_info.path, **diffusers_load_config) + elif hasattr(pipeline, 'from_ckpt'): + sd_model = pipeline.from_ckpt(checkpoint_info.path, **diffusers_load_config) + else: + shared.log.error(f'Diffusers cannot load safetensor model: {checkpoint_info.path} {shared.opts.diffusers_pipeline}') + return except Exception as e: shared.log.error(f'Diffusers failed loading model using pipeline: {checkpoint_info.path} {shared.opts.diffusers_pipeline} {e}') return @@ -938,13 +962,14 @@ def unload_model_weights(op='model'): if shared.backend == shared.Backend.ORIGINAL: sd_hijack.model_hijack.undo_hijack(model_data.sd_model) model_data.sd_model = None + shared.log.debug(f'Weights unloaded {op}: {memory_stats()}') else: if model_data.sd_refiner: model_data.sd_refiner.to(devices.cpu) if shared.backend == shared.Backend.ORIGINAL: sd_hijack.model_hijack.undo_hijack(model_data.sd_refiner) model_data.sd_refiner = None - shared.log.debug(f'Weights unloaded {op}: {memory_stats()}') + shared.log.debug(f'Weights unloaded {op}: {memory_stats()}') devices.torch_gc(force=True) diff --git a/modules/sd_vae.py b/modules/sd_vae.py index 1943ec5f9..8ad295ccf 100644 --- a/modules/sd_vae.py +++ b/modules/sd_vae.py @@ -5,6 +5,7 @@ from copy import deepcopy import torch from modules import shared, paths, devices, script_callbacks, sd_models + vae_ignore_keys = {"model_ema.decay", "model_ema.num_updates"} vae_dict = {} base_vae = None @@ -13,6 +14,7 @@ checkpoint_info = None vae_path = os.path.abspath(os.path.join(paths.models_path, 'VAE')) checkpoints_loaded = collections.OrderedDict() + def get_base_vae(model): if base_vae is not None and checkpoint_info == model.sd_checkpoint_info and model: return base_vae @@ -147,6 +149,26 @@ def load_vae(model, vae_file=None, vae_source="from unknown source"): loaded_vae_file = vae_file +def load_vae_diffusers(_model, vae_file=None, vae_source="from unknown source"): + global loaded_vae_file # pylint: disable=global-statement + if loaded_vae_file == vae_file: + return + loaded_vae_file = None + if vae_file is None: + return + if not os.path.isfile(vae_file): + shared.log.error('VAE not found: {vae_file}') + return + shared.log.info(f"Loading diffusers VAE: {vae_source}: {vae_file}") + try: + import diffusers + diffusers_vae = diffusers.AutoencoderKL.from_pretrained(vae_file) + except Exception as e: + shared.log.error(f"Loading diffusers VAE failed: {vae_file} {e}") + diffusers_vae = None + return diffusers_vae + + # don't call this from outside def _load_vae_dict(model, vae_dict_1): model.first_stage_model.load_state_dict(vae_dict_1) @@ -178,12 +200,17 @@ def reload_vae_weights(sd_model=None, vae_file=unspecified): lowvram.send_everything_to_cpu() else: sd_model.to(devices.cpu) - sd_hijack.model_hijack.undo_hijack(sd_model) - if shared.cmd_opts.rollback_vae and devices.dtype_vae == torch.bfloat16: - devices.dtype_vae = torch.float16 - load_vae(sd_model, vae_file, vae_source) - sd_hijack.model_hijack.hijack(sd_model) - script_callbacks.model_loaded_callback(sd_model) + + if shared.backend == shared.Backend.ORIGINAL: + sd_hijack.model_hijack.undo_hijack(sd_model) + if shared.cmd_opts.rollback_vae and devices.dtype_vae == torch.bfloat16: + devices.dtype_vae = torch.float16 + load_vae(sd_model, vae_file, vae_source) + sd_hijack.model_hijack.hijack(sd_model) + script_callbacks.model_loaded_callback(sd_model) + elif shared.backend == shared.Backend.DIFFUSERS: + load_vae_diffusers(sd_model, vae_file, vae_source) + if not shared.cmd_opts.lowvram and not shared.cmd_opts.medvram: sd_model.to(devices.device) shared.log.info(f"VAE weights loaded: {vae_file}") diff --git a/modules/shared.py b/modules/shared.py index a0aa8be8d..f654c69bc 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -38,7 +38,10 @@ hypernetworks = {} loaded_hypernetworks = [] gradio_theme = gr.themes.Base() settings_components = None -pipelines = ['Stable Diffusion', 'Stable Diffusion XL', 'Kandinsky V1', 'Kandinsky V2', 'DeepFloyd IF', 'Shap-E'] +pipelines = [ + 'Stable Diffusion', 'Stable Diffusion XL', 'Kandinsky V1', 'Kandinsky V2', 'DeepFloyd IF', 'Shap-E', + 'Stable Diffusion Img2Img', 'Stable Diffusion XL Img2Img', 'Kandinsky V1 Img2Img', 'Kandinsky V2 Img2Img', 'DeepFloyd IF Img2Img', 'Shap-E Img2Img' +] latent_upscale_default_mode = "Latent" latent_upscale_modes = { "Latent": {"mode": "bilinear", "antialias": False}, @@ -356,7 +359,8 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { })) options_templates.update(options_section(('diffusers', "Diffusers Settings"), { - "diffusers_pipeline": OptionInfo(pipelines[0], 'Diffuser Pipeline', gr.Dropdown, lambda: {"choices": pipelines}), + "diffusers_allow_safetensors": OptionInfo(False, 'Diffuser Pipeline when loading from safetensors'), + "diffusers_pipeline": OptionInfo(pipelines[0], 'Diffuser Pipeline when loading from safetensors', gr.Dropdown, lambda: {"choices": pipelines}), "diffusers_extract_ema": OptionInfo(True, "Use model EMA weights when possible"), "diffusers_generator_device": OptionInfo("default", "Generator device", gr.Radio, lambda: {"choices": ["default", "cpu"]}), "diffusers_seq_cpu_offload": OptionInfo(False, "Enable sequential CPU offload"), diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index cfb4e7525..c893b6de0 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -66,6 +66,7 @@ class ExtraNetworksPage: self.allow_negative_prompt = False self.metadata = {} self.info = {} + self.html = '' self.items = [] self.missing_thumbs = [] self.card = ''' @@ -150,7 +151,6 @@ class ExtraNetworksPage: self_name_id = self.name.replace(" ", "_") if skip: return f"
    Extra network page not ready
    Click refresh to try again
    " - items_html = '' subdirs = {} allowed_folders = [os.path.abspath(x) for x in self.allowed_directories_for_previews()] for parentdir in [*set(allowed_folders)]: @@ -174,16 +174,21 @@ class ExtraNetworksPage: {html.escape(subdir) if subdir!="" else "all"}
    """ for subdir in subdirs]) try: + if len(self.html) > 0: + res = f"
    {subdirs_html}
    {self.html}
    " + return res + self.html = '' self.items = list(self.list_items()) self.create_xyz_grid() for item in self.items: self.metadata[item["name"]] = item.get("metadata", {}) self.info[item["name"]] = self.find_info(item['filename']) - items_html += self.create_html_for_item(item, tabname) - if len(subdirs_html) > 0 or len(items_html) > 0: - res = f"
    {subdirs_html}
    {items_html}
    " + self.html += self.create_html_for_item(item, tabname) + if len(subdirs_html) > 0 or len(self.html) > 0: + res = f"
    {subdirs_html}
    {self.html}
    " else: return '' + shared.log.debug(f'Extra networks: {self.name} items={len(self.items)} subdirs={len(subdirs)}') threading.Thread(target=self.create_thumb).start() return res except Exception as e: @@ -327,6 +332,7 @@ def create_ui(container, button, tabname, skip_indexing = False): def refresh(): res = [] for pg in ui.stored_extra_pages: + pg.html = '' pg.refresh() res.append(pg.create_html(ui.tabname)) ui.search.update(value = ui.search.value) From 3e4ca0095e2051886125d5532e2157497c652dc3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 7 Jul 2023 10:05:32 -0400 Subject: [PATCH 32/42] fix compile --- modules/processing.py | 2 -- modules/sd_models.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index 04058d519..1ea9ee251 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -710,8 +710,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: shared.state.sampling_steps = p.steps shared.state.current_latent = latents shared.state.set_current_image() - if p.scripts is not None: - p.scripts.process(p) output = shared.sd_model( # pylint: disable=not-callable prompt=prompts, diff --git a/modules/sd_models.py b/modules/sd_models.py index 688aa3638..ae940ad61 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -563,6 +563,7 @@ class PriorPipeline: def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument + import torch # todo: no idea why its undefined here if timer is None: timer = Timer() import logging From 1c22722c8caef017ef256f6e9e2743a2121ae693 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 7 Jul 2023 10:10:53 -0400 Subject: [PATCH 33/42] enable sampler swap for diffusers --- modules/processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/processing.py b/modules/processing.py index 1ea9ee251..fd3adebef 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -691,7 +691,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") - # shared.sd_model.scheduler = sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op + shared.sd_model.scheduler = sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op cross_attention_kwargs={} if lora_state['active']: From bf84ee8168f6fc784f2df71f372a953d9eeb0ec3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 7 Jul 2023 12:48:27 -0400 Subject: [PATCH 34/42] update diffusers --- DIFFUSERS.md | 8 +++++--- modules/processing.py | 10 ++++++++++ requirements.txt | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index df2c462fc..db052527c 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -21,6 +21,8 @@ All notes have moved to [Wiki page](https://github.com/vladmandic/automatic/wiki ## TODO - VAE - > vae = AutoencoderKL.from_pretrained("stabilityai/sdxl-vae") - > pipe = StableDiffusionPipeline.from_pretrained(model, vae=vae) -- Refiner handler with medvram/lowvram +- Refiner with medvram/lowvram +- SD-XL from safetensors +- Hires fix +- Callbacks +- Stop/Skip diff --git a/modules/processing.py b/modules/processing.py index fd3adebef..9ebd20ede 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -711,6 +711,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: shared.state.current_latent = latents shared.state.set_current_image() + # shared.sd_model.to(devices.device) output = shared.sd_model( # pylint: disable=not-callable prompt=prompts, negative_prompt=negative_prompts, @@ -723,8 +724,13 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: cross_attention_kwargs=cross_attention_kwargs, **task_specific_kwargs ) + # if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: + # shared.sd_model.to('cpu') + # devices.torch_gc(force=True) + if shared.sd_refiner is not None: + # shared.sd_refiner.to(devices.device) init_image = output.images[0] output = shared.sd_refiner( # pylint: disable=not-callable prompt=prompts, @@ -738,6 +744,10 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: cross_attention_kwargs=cross_attention_kwargs, image=init_image ) + # if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: + # shared.sd_refiner.to('cpu') + # devices.torch_gc(force=True) + x_samples_ddim = output.images diff --git a/requirements.txt b/requirements.txt index 6dc79e46a..12add323f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,7 +49,7 @@ requests==2.31.0 tqdm==4.65.0 accelerate==0.20.3 opencv-python==4.7.0.72 -diffusers==0.18.0 +diffusers==0.18.1 einops==0.4.1 gradio==3.32.0 numexpr==2.8.4 From 120710f28a65c313b3cc749a04565ce00c761d17 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 7 Jul 2023 13:38:04 -0400 Subject: [PATCH 35/42] force model variant --- modules/sd_models.py | 5 ++++- modules/sd_vae.py | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index ae940ad61..ec47c7140 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -563,7 +563,7 @@ class PriorPipeline: def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument - import torch # todo: no idea why its undefined here + import torch # pylint: disable=reimported,redefined-outer-name if timer is None: timer = Timer() import logging @@ -577,6 +577,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No "load_safety_checker": False, # "use_safetensors": True, # TODO(PVP) - we can't enable this for all checkpoints just yet } + if devices.dtype == torch.float16: + diffusers_load_config['variant'] = 'fp16' if shared.opts.data.get('sd_model_checkpoint', '') == 'model.ckpt' or shared.opts.data.get('sd_model_checkpoint', '') == '': shared.opts.data['sd_model_checkpoint'] = "runwayml/stable-diffusion-v1-5" @@ -588,6 +590,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if model_data.sd_refiner is not None and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model return + shared.log.debug(f'Diffusers load config: {diffusers_load_config}') sd_model = None try: devices.set_cuda_params() diff --git a/modules/sd_vae.py b/modules/sd_vae.py index 8ad295ccf..38ffde339 100644 --- a/modules/sd_vae.py +++ b/modules/sd_vae.py @@ -160,9 +160,17 @@ def load_vae_diffusers(_model, vae_file=None, vae_source="from unknown source"): shared.log.error('VAE not found: {vae_file}') return shared.log.info(f"Loading diffusers VAE: {vae_source}: {vae_file}") + diffusers_load_config = { + "low_cpu_mem_usage": True, + "torch_dtype": devices.dtype_vae, + "use_safetensors": True, + } + if devices.dtype_vae == torch.float16: + diffusers_load_config['variant'] = 'fp16' + shared.log.debug(f'Diffusers VAE load config: {diffusers_load_config}') try: import diffusers - diffusers_vae = diffusers.AutoencoderKL.from_pretrained(vae_file) + diffusers_vae = diffusers.AutoencoderKL.from_pretrained(vae_file, **diffusers_load_config) except Exception as e: shared.log.error(f"Loading diffusers VAE failed: {vae_file} {e}") diffusers_vae = None From 4459cc581a2df6cad57883b63fee9c144260830c Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 7 Jul 2023 23:03:29 +0300 Subject: [PATCH 36/42] Prior device.type cuda or xpu --- modules/sd_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index ec47c7140..505d79101 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -546,7 +546,7 @@ class PriorPipeline: def __call__(self, *args, **kwargs): unclip_outputs = self.prior(prompt=kwargs.get("prompt"), negative_prompt=kwargs.get("negative_prompt")) - if self.prior.device.type == "cuda": + if self.prior.device.type == "cuda" or self.prior.device.type == "xpu": prior_device = self.prior.device self.prior.to("cpu") self.main.to(prior_device) @@ -554,7 +554,7 @@ class PriorPipeline: kwargs = {**kwargs, **unclip_outputs} result = self.main(*args, **kwargs) - if self.main.device.type == "cuda": + if self.main.device.type == "cuda" or self.main.device.type == "xpu": main_device = self.main.device self.main.to("cpu") self.prior.to(main_device) From 0f4f8c60153179eb76fccaea2b689e77ab04530c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 7 Jul 2023 20:20:45 -0400 Subject: [PATCH 37/42] extra networks fixes --- installer.py | 1 + javascript/extraNetworks.js | 56 +++++++++++++++++------------------- javascript/imageViewer.js | 12 ++++---- javascript/style.css | 5 ++-- modules/shared.py | 4 ++- modules/ui.py | 22 +++++++++----- modules/ui_extra_networks.py | 3 +- 7 files changed, 56 insertions(+), 47 deletions(-) diff --git a/installer.py b/installer.py index d54e95eb8..01f30a523 100644 --- a/installer.py +++ b/installer.py @@ -105,6 +105,7 @@ def setup_logging(): logging.getLogger("httpx").setLevel(logging.ERROR) logging.getLogger("ControlNet").handlers = log.handlers logging.getLogger("lycoris").handlers = log.handlers + # logging.getLogger("DeepSpeed").handlers = log.handlers def print_profile(profile: cProfile.Profile, msg: str): diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index d4dfc243e..5f02ed586 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -26,37 +26,33 @@ function setupExtraNetworksForTab(tabname) { }); intersectionObserver = new IntersectionObserver((entries) => { + if (!en) return + for (el of Array.from(gradioApp().querySelectorAll('.extra-network-cards'))) el.style.height = window.opts.extra_networks_height + 'vh'; if (entries[0].intersectionRatio > 0) { - for (el of Array.from(gradioApp().querySelectorAll('.extra-network-cards'))) { - const rect = el.getBoundingClientRect(); - if (rect.top > 0) { - if (!en) return - if (window.opts.extra_networks_card_cover == 'cover') { - en.style.transition = ''; - en.style.zIndex = 9999; - en.style.position = 'absolute'; - en.style.right = 'unset'; - en.style.width = 'unset'; - el.style.height = document.body.offsetHeight - el.getBoundingClientRect().top + 'px'; - gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' - } if (window.opts.extra_networks_card_cover == 'sidebar') { - en.style.transition = 'width 0.2s ease'; - en.style.zIndex = 0; - en.style.position = 'absolute'; - en.style.right = '0'; - en.style.width = window.opts.extra_networks_sidebar_width + 'vw'; - el.style.height = gradioApp().getElementById(`${tabname}_settings`).offsetHeight - 90 + 'px'; - gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 100 - 2 - window.opts.extra_networks_sidebar_width + 'vw'; - } else { - en.style.transition = ''; - en.style.zIndex = 0; - en.style.position = 'relative'; - en.style.right = 'unset'; - en.style.width = 'unset'; - el.style.height = window.innerHeight - el.getBoundingClientRect().top + 'px'; - gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' - } - } + if (window.opts.extra_networks_card_cover === 'cover') { + en.style.transition = ''; + en.style.zIndex = 9999; + en.style.position = 'absolute'; + en.style.right = 'unset'; + en.style.width = 'unset'; + en.style.height = 'unset'; + gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' + } else if (window.opts.extra_networks_card_cover === 'sidebar') { + en.style.transition = 'width 0.2s ease'; + en.style.zIndex = 0; + en.style.position = 'absolute'; + en.style.right = '0'; + en.style.width = window.opts.extra_networks_sidebar_width + 'vw'; + en.style.height = '-webkit-fill-available' + gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 100 - 2 - window.opts.extra_networks_sidebar_width + 'vw'; + } else { + en.style.transition = ''; + en.style.zIndex = 0; + en.style.position = 'relative'; + en.style.right = 'unset'; + en.style.width = 'unset'; + en.style.height = 'unset'; + gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' } } else { en.style.width = 0; diff --git a/javascript/imageViewer.js b/javascript/imageViewer.js index 4a5302fcc..b046b0503 100644 --- a/javascript/imageViewer.js +++ b/javascript/imageViewer.js @@ -150,7 +150,6 @@ onAfterUiUpdate(() => { document.addEventListener('DOMContentLoaded', () => { const modal = document.createElement('div'); - // modal.onclick = closeModal; modal.id = 'lightboxModal'; modal.tabIndex = 0; modal.addEventListener('keydown', modalKeyHandler, true); @@ -187,12 +186,13 @@ document.addEventListener('DOMContentLoaded', () => { modalImage.addEventListener('keydown', modalKeyHandler, true); modal.appendChild(modalImage); modalImage.onload = () => panzoom(modalImage, { zoomSpeed: 0.025, minZoom: 0.25, maxZoom: 4.0 }); - let drag = false; - modalImage.addEventListener('mousedown', () => drag = false); - modalImage.addEventListener('mousemove', () => drag = true); - modalImage.addEventListener('mouseup', () => { if (!drag) closeModal(); }); - // modalImage.onclick = closeModal; + let drag = false; + modal.addEventListener('mousedown', () => drag = false); + modal.addEventListener('mousemove', () => drag = true); + modal.addEventListener('scroll', () => drag = true); + modal.addEventListener('mouseup', () => { if (!drag) closeModal(); }); + const modalPrev = document.createElement('a'); modalPrev.className = 'modalPrev'; modalPrev.innerHTML = '❮'; diff --git a/javascript/style.css b/javascript/style.css index 1a09def29..3487c461a 100644 --- a/javascript/style.css +++ b/javascript/style.css @@ -432,6 +432,7 @@ div#extras_scale_to_tab div.form{ height: 100%; width: 100%; min-height: 0; + background: transparent; } table.settings-value-table{ @@ -533,11 +534,11 @@ table.settings-value-table td{ .extra-networks .description { margin-top: 8px; } .extra-networks .tab-nav > button { margin-right: 0; height: auto; padding: 2px 4px 2px 4px; } .extra-networks-tab { padding: 0 !important; } -.extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; max-height: 50vh; min-width: 80px; max-width: 120px; } +.extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; min-width: 80px; max-width: 120px; } .extra-networks-page { display: flex } .extra-networks .custom-button { min-width: 80px; max-width: 120px; width: 100%; background: none; justify-content: left; text-align: left; padding: 2px 8px 2px 8px; box-shadow: none; line-break: auto; } .extra-networks .custom-button:hover { background: var(--button-primary-background-fill) } -.extra-network-cards { display: flex; flex-wrap: wrap; height: 50vh; max-height: 50vh; overflow-y: scroll; overflow-x: hidden; width: -webkit-fill-available; } +.extra-network-cards { display: flex; flex-wrap: wrap; overflow-y: scroll; overflow-x: hidden; width: -webkit-fill-available; } .extra-network-cards .card { height: fit-content; margin: 0.5em; position: relative; scroll-snap-align: start; scroll-margin-top: 0; } .extra-network-cards .card .overlay { position: absolute; bottom: 0; padding: 0.2em; z-index: 10; width: 100%; background: none; } .extra-network-cards .card:hover .overlay { background: rgba(0, 0, 0, 0.40); } diff --git a/modules/shared.py b/modules/shared.py index f654c69bc..959ab7851 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -589,9 +589,11 @@ options_templates.update(options_section(('upscaling', "Upscaling"), { options_templates.update(options_section(('extra_networks', "Extra Networks"), { "ui_extra_networks_tab_reorder": OptionInfo("Checkpoints, Lora, LyCORIS, Textual Inversion, Hypernetworks", "Extra networks tab order"), "extra_networks_card_cover": OptionInfo("inline", "UI position", gr.Radio, lambda: {"choices": ["cover", "inline", "sidebar"]}), + "extra_networks_height": OptionInfo(47, "UI height (%)", gr.Slider, {"minimum": 10, "maximum": 100, "step": 1}), "extra_networks_sidebar_width": OptionInfo(35, "UI sidebar width (%)", gr.Slider, {"minimum": 10, "maximum": 80, "step": 1}), + "extra_networks_card_lazy": OptionInfo(True, "UI card preview lazy loading"), "extra_networks_card_size": OptionInfo(200, "UI card size (px)", gr.Slider, {"minimum": 20, "maximum": 2000, "step": 1}), - "extra_networks_card_square": OptionInfo(False, "UI disable variable aspect ratio"), + "extra_networks_card_square": OptionInfo(True, "UI disable variable aspect ratio"), "extra_networks_card_fit": OptionInfo("cover", "UI image contain method", gr.Radio, lambda: {"choices": ["contain", "cover", "fill"]}), "extra_network_skip_indexing": OptionInfo(False, "Do not automatically build extra network pages", gr.Checkbox), "lyco_patch_lora": OptionInfo(False, "Use LyCoris handler for all Lora types", gr.Checkbox), diff --git a/modules/ui.py b/modules/ui.py index 37b6c0363..3de32c239 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -13,7 +13,7 @@ from modules.call_queue import wrap_gradio_gpu_call, wrap_queued_call, wrap_grad from modules import sd_hijack, sd_models, script_callbacks, ui_extensions, deepbooru, extra_networks, ui_common, ui_postprocessing, ui_loadsave, ui_train, ui_models from modules.ui_components import FormRow, FormColumn, FormGroup, ToolButton, FormHTML # pylint: disable=unused-import from modules.paths import script_path, data_path -from modules.shared import opts, cmd_opts, backend, Backend +from modules.shared import opts, cmd_opts from modules import prompt_parser import modules.codeformer_model import modules.generation_parameters_copypaste as parameters_copypaste @@ -198,13 +198,21 @@ def update_token_counter(text, steps): prompt_schedules = [[[steps, text]]] flat_prompts = reduce(lambda list1, list2: list1+list2, prompt_schedules) prompts = [prompt_text for step, prompt_text in flat_prompts] - if backend == Backend.ORIGINAL: + if modules.shared.backend == modules.shared.Backend.ORIGINAL: token_count, max_length = max([sd_hijack.model_hijack.get_prompt_lengths(prompt) for prompt in prompts], key=lambda args: args[0]) - else: - tokenizer = modules.shared.sd_model.tokenizer - has_bos_token, has_eos_token = tokenizer.bos_token_id is not None, tokenizer.eos_token_id is not None - token_count = max([len(modules.shared.sd_model.tokenizer(prompt)) for prompt in prompts]) - int(has_bos_token) - int(has_eos_token) - max_length = tokenizer.model_max_length - int(has_bos_token) - int(has_eos_token) + elif modules.shared.backend == modules.shared.Backend.DIFFUSERS: + if modules.shared.sd_model is not None: + tokenizer = modules.shared.sd_model.tokenizer + has_bos_token = tokenizer.bos_token_id is not None + has_eos_token = tokenizer.eos_token_id is not None + ids = [modules.shared.sd_model.tokenizer(prompt) for prompt in prompts] + if len(ids) > 0 and hasattr(ids[0], 'input_ids'): + ids = [x.input_ids for x in ids] + token_count = max([len(x) for x in ids]) - int(has_bos_token) - int(has_eos_token) + max_length = tokenizer.model_max_length - int(has_bos_token) - int(has_eos_token) + else: + token_count = 0 + max_length = 75 return f"{token_count}/{max_length}" diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index c893b6de0..e2756917c 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -83,7 +83,7 @@ class ExtraNetworksPage: ℹ️ - + ''' # noqa: RUF001 @@ -214,6 +214,7 @@ class ExtraNetworksPage: "name": item["name"], "description": (item.get("description") or ""), "search_term": item.get("search_term", ""), + "loading": "lazy" if shared.opts.extra_networks_card_lazy else "eager", "card_click": item.get("onclick", '"' + html.escape(f"""return cardClicked({json.dumps(tabname)}, {item.get("prompt", None)}, {"true" if self.allow_negative_prompt else "false"})""") + '"'), "card_save_desc": '"' + html.escape(f"""return saveCardDescription(event, {json.dumps(tabname)}, {json.dumps(item["local_preview"])})""") + '"', "card_save_preview": '"' + html.escape(f"""return saveCardPreview(event, {json.dumps(tabname)}, {json.dumps(item["local_preview"])})""") + '"', From 5bf3d229d0eaea67ff20eaec631b8c0975f0a654 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 7 Jul 2023 21:54:21 -0400 Subject: [PATCH 38/42] rehost clip-interrogator and update installer --- .gitmodules | 2 +- TODO.md | 4 +- extensions-builtin/LDSR/sd_hijack_ddpm_v1.py | 2 +- extensions-builtin/a1111-sd-webui-lycoris | 2 +- extensions-builtin/clip-interrogator-ext | 2 +- extensions-builtin/sd-extension-system-info | 2 +- extensions-builtin/sd-webui-agent-scheduler | 2 +- extensions-builtin/sd-webui-controlnet | 2 +- installer.py | 54 +++++++++++--------- modules/models/diffusion/ddpm_edit.py | 9 ++-- requirements.txt | 2 +- scripts/outpainting_mk_2.py | 2 +- wiki | 2 +- 13 files changed, 48 insertions(+), 39 deletions(-) diff --git a/.gitmodules b/.gitmodules index 93009f907..6b435fe50 100644 --- a/.gitmodules +++ b/.gitmodules @@ -18,7 +18,7 @@ ignore = dirty [submodule "extensions-builtin/clip-interrogator-ext"] path = extensions-builtin/clip-interrogator-ext - url = https://github.com/pharmapsychotic/clip-interrogator-ext.git + url = https://github.com/Dahvikiin/clip-interrogator-ext.git ignore = dirty [submodule "extensions-builtin/sd-webui-controlnet"] path = extensions-builtin/sd-webui-controlnet diff --git a/TODO.md b/TODO.md index f4da3031c..e2f68a7de 100644 --- a/TODO.md +++ b/TODO.md @@ -60,4 +60,6 @@ Tech that can be integrated as part of the core workflow... - git-rebasin - additional upscalers - new image browser -- fp8 +- `fp8` +- update `transformers` +- `git submodule set-url extensions-builtin/clip-interrogator-ext https://github.com/Dahvikiin/clip-interrogator-ext.git` diff --git a/extensions-builtin/LDSR/sd_hijack_ddpm_v1.py b/extensions-builtin/LDSR/sd_hijack_ddpm_v1.py index 631a08ef0..bc2af045a 100644 --- a/extensions-builtin/LDSR/sd_hijack_ddpm_v1.py +++ b/extensions-builtin/LDSR/sd_hijack_ddpm_v1.py @@ -958,7 +958,7 @@ class LatentDiffusionV1(DDPMV1): cond_list = [{'c_crossattn': [e]} for e in adapted_cond] else: - cond_list = [cond for i in range(z.shape[-1])] # Todo make this more efficient + cond_list = [cond for i in range(z.shape[-1])] # apply model by loop over crops output_list = [self.model(z_list[i], t, **cond_list[i]) for i in range(z.shape[-1])] diff --git a/extensions-builtin/a1111-sd-webui-lycoris b/extensions-builtin/a1111-sd-webui-lycoris index 123d1da15..025dea967 160000 --- a/extensions-builtin/a1111-sd-webui-lycoris +++ b/extensions-builtin/a1111-sd-webui-lycoris @@ -1 +1 @@ -Subproject commit 123d1da15d802823480f8020312ce449523f10e2 +Subproject commit 025dea96720197dd4486a5bb8e2f4d72a95a3088 diff --git a/extensions-builtin/clip-interrogator-ext b/extensions-builtin/clip-interrogator-ext index c0bf90052..6e31272e1 160000 --- a/extensions-builtin/clip-interrogator-ext +++ b/extensions-builtin/clip-interrogator-ext @@ -1 +1 @@ -Subproject commit c0bf90052a14a104b2dbfd9b7e7818aae5ca5ae1 +Subproject commit 6e31272e14308b4918f9785b1dda7cc1149e8838 diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index 5b13bfeee..9433a15b8 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit 5b13bfeeebee1fc984bfde4e3171b31e4eee5a6b +Subproject commit 9433a15b8965f9b0aa6a93cd44992f29de3f3e79 diff --git a/extensions-builtin/sd-webui-agent-scheduler b/extensions-builtin/sd-webui-agent-scheduler index 8198489fd..75ba093d4 160000 --- a/extensions-builtin/sd-webui-agent-scheduler +++ b/extensions-builtin/sd-webui-agent-scheduler @@ -1 +1 @@ -Subproject commit 8198489fd42883d3af119e219c320ba4853802c8 +Subproject commit 75ba093d46b37f2d01fa4207680a4fa6417b01c3 diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 2785cbe61..dd766de86 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 2785cbe61a61c137d4e980752771f6329b03612a +Subproject commit dd766de8629ee6035a734217e08c26cd1b08b2ab diff --git a/installer.py b/installer.py index 01f30a523..a66b399aa 100644 --- a/installer.py +++ b/installer.py @@ -208,25 +208,32 @@ def git(arg: str, folder: str = None, ignore: bool = False): log.debug(f'Git output: {txt}') return txt - -# update switch to main branch as head can get detached and update repository -def update(folder): +# switch to main branch as head can get detached +def branch(folder): if not os.path.exists(os.path.join(folder, '.git')): return - branch = git('branch', folder) - if 'main' in branch: - branch = 'main' - elif 'master' in branch: - branch = 'master' + b = git('branch', folder) + if 'main' in b: + b = 'main' + elif 'master' in b: + b = 'master' else: - branch = branch.split('\n')[0].replace('*', '').strip() - # log.debug(f'Setting branch: {folder} / {branch}') - git(f'checkout {branch}', folder) + b = b.split('\n')[0].replace('*', '').strip() + log.debug(f'Submodule: {folder} / {b}') + git(f'checkout {b}', folder, ignore=True) + return b + + +# update git repository +def update(folder, current_branch = False): + if current_branch: + git('pull --autostash --rebase --force', folder) + return + b = branch(folder) if branch is None: git('pull --autostash --rebase --force', folder) else: - git(f'pull origin {branch} --autostash --rebase --force', folder) - # branch = git('branch', folder) + git(f'pull origin {b} --autostash --rebase --force', folder) # clone git repository @@ -528,7 +535,7 @@ def install_submodules(): pr.enable() log.info('Verifying submodules') txt = git('submodule') - log.debug(f'Submodules list: {txt}') + # log.debug(f'Submodules list: {txt}') if 'no submodule mapping found' in txt: log.warning('Attempting repository recover') git('add .') @@ -540,15 +547,16 @@ def install_submodules(): txt = git('submodule') log.info('Continuing setup') git('submodule --quiet update --init --recursive') - if args.upgrade: - log.info('Updating submodules') - submodules = txt.splitlines() - for submodule in submodules: - try: - name = submodule.split()[1].strip() + submodules = txt.splitlines() + for submodule in submodules: + try: + name = submodule.split()[1].strip() + if args.upgrade: update(name) - except Exception: - log.error(f'Error updating submodule: {submodule}') + else: + branch(name) + except Exception: + log.error(f'Error updating submodule: {submodule}') if args.profile: print_profile(pr, 'Submodule') @@ -653,7 +661,7 @@ def check_version(offline=False, reset=True): # pylint: disable=unused-argument try: git('add .') git('stash') - update('.') + update('.', current_branch=True) # git('git stash pop') ver = git('log -1 --pretty=format:"%h %ad"') log.info(f'Upgraded to version: {ver}') diff --git a/modules/models/diffusion/ddpm_edit.py b/modules/models/diffusion/ddpm_edit.py index c847bfd68..72e011d6f 100644 --- a/modules/models/diffusion/ddpm_edit.py +++ b/modules/models/diffusion/ddpm_edit.py @@ -631,7 +631,7 @@ class LatentDiffusion(DDPM): weighting = weighting * L_weighting return weighting - def get_fold_unfold(self, x, kernel_size, stride, uf=1, df=1): # todo load once not every time, shorten code + def get_fold_unfold(self, x, kernel_size, stride, uf=1, df=1): """ :param x: img of size (bs, c, h, w) :return: n img crops of size (n, bs, c, kernel_size[0], kernel_size[1]) @@ -919,7 +919,7 @@ class LatentDiffusion(DDPM): z_list = [z[:, :, :, :, i] for i in range(z.shape[-1])] if self.cond_stage_key in ["image", "LR_image", "segmentation", - 'bbox_img'] and self.model.conditioning_key: # todo check for completeness + 'bbox_img'] and self.model.conditioning_key: c_key = next(iter(cond.keys())) # get key c = next(iter(cond.values())) # get value assert (len(c) == 1) # todo extend to list with more than one elem @@ -973,12 +973,11 @@ class LatentDiffusion(DDPM): cond_list = [{'c_crossattn': [e]} for e in adapted_cond] else: - cond_list = [cond for i in range(z.shape[-1])] # Todo make this more efficient + cond_list = [cond for i in range(z.shape[-1])] # apply model by loop over crops output_list = [self.model(z_list[i], t, **cond_list[i]) for i in range(z.shape[-1])] - assert not isinstance(output_list[0], - tuple) # todo cant deal with multiple model outputs check this never happens + assert not isinstance(output_list[0], tuple) o = torch.stack(output_list, axis=-1) o = o * weighting diff --git a/requirements.txt b/requirements.txt index 12add323f..dd1ada5a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -58,7 +58,7 @@ numba==0.57.0 pandas==1.5.3 protobuf==3.20.3 pytorch_lightning==1.9.4 -transformers==4.26.1 +transformers==4.30.2 timm==0.6.13 tomesd==0.1.3 urllib3==1.26.15 diff --git a/scripts/outpainting_mk_2.py b/scripts/outpainting_mk_2.py index bf1daf55f..4b0af3053 100644 --- a/scripts/outpainting_mk_2.py +++ b/scripts/outpainting_mk_2.py @@ -99,7 +99,7 @@ def get_matched_noise(_np_src_image, np_mask_rgb, noise_q=1, color_variation=0.0 shaped_noise_fft = _fft2(noise_rgb) shaped_noise_fft[:, :, :] = np.absolute(shaped_noise_fft[:, :, :]) ** 2 * (src_dist ** noise_q) * src_phase # perform the actual shaping - brightness_variation = 0. # color_variation # todo: temporarily tieing brightness variation to color variation for now + brightness_variation = 0. # color_variation contrast_adjusted_np_src = _np_src_image[:] * (brightness_variation + 1.) - brightness_variation * 2. # scikit-image is used for histogram matching, very convenient! diff --git a/wiki b/wiki index 28e3cc15e..503fa982c 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 28e3cc15ef4566564764fa73542ab6b00d2b0959 +Subproject commit 503fa982cba22a33f3601f43fd255883167e421c From 816876c8ac2faec21528210419d5ecb92c2ebfdb Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 7 Jul 2023 21:59:50 -0400 Subject: [PATCH 39/42] fix installer errors --- installer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/installer.py b/installer.py index a66b399aa..07de36730 100644 --- a/installer.py +++ b/installer.py @@ -200,6 +200,8 @@ def git(arg: str, folder: str = None, ignore: bool = False): txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore") txt = txt.strip() if result.returncode != 0 and not ignore: + if "couldn't find remote ref" in txt: # not a git repo + return txt global errors # pylint: disable=global-statement errors += 1 log.error(f'Error running git: {folder} / {arg}') @@ -683,7 +685,6 @@ def update_wiki(): log.info('Updating Wiki') try: update(os.path.join(os.path.dirname(__file__), "wiki")) - update(os.path.join(os.path.dirname(__file__), "wiki", "origin-wiki")) except Exception: log.error('Error updating wiki') From 89a7ea6a3f2ebea065e3e1203c166b1fa984f523 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 8 Jul 2023 09:49:41 -0400 Subject: [PATCH 40/42] overal quality fixes --- CHANGELOG.md | 6 ++-- DIFFUSERS.md | 28 ----------------- extensions-builtin/LDSR/sd_hijack_ddpm_v1.py | 3 +- installer.py | 4 ++- javascript/black-orange.css | 33 +++++++++++--------- modules/devices.py | 15 +++++++++ modules/lora_diffusers.py | 2 +- modules/modelloader.py | 5 ++- modules/models/diffusion/ddpm_edit.py | 4 +-- modules/processing.py | 21 +++++++------ modules/sd_models.py | 13 ++++---- modules/ui_models.py | 4 +-- requirements.txt | 2 +- wiki | 2 +- 14 files changed, 68 insertions(+), 74 deletions(-) delete mode 100644 DIFFUSERS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 18d85957b..dfdf9f26d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,13 @@ # Change Log for SD.Next -## Update for 07/07/2023 +## Update for 07/08/2023 - add pan & zoom controls (touch and mouse) to image viewer (lightbox) +- cache extra networks between tabs + this should result in neat 2x speedup on building extra networks - add settings -> extra networks -> do not automatically build extra network pages speeds up app start if you have a lot of extra networks and you want to build them manually when needed - extra network ui tweaks -- cache extra networks between tabs - this should result in neat 2x speedup on building extra networks - merge experimental diffusers support covered in details in a separate post diff --git a/DIFFUSERS.md b/DIFFUSERS.md deleted file mode 100644 index db052527c..000000000 --- a/DIFFUSERS.md +++ /dev/null @@ -1,28 +0,0 @@ -# Diffusers - -## Install - -initial support merged into `dev` branch - -- download from branch and start as normal: - > git clone https://github.com/vladmandic/automatic -b dev diffusers - > cd diffusers - > webui --debug --backend diffusers - -- to go back to standard execution pipeline, start with - > webui --debug --backend original - -- To update repo, do not use `--upgrade` flag, use manual `git pull` instead - -## Notes - -All notes have moved to [Wiki page](https://github.com/vladmandic/automatic/wiki/Diffusers) - -## TODO - -- VAE -- Refiner with medvram/lowvram -- SD-XL from safetensors -- Hires fix -- Callbacks -- Stop/Skip diff --git a/extensions-builtin/LDSR/sd_hijack_ddpm_v1.py b/extensions-builtin/LDSR/sd_hijack_ddpm_v1.py index bc2af045a..053be8290 100644 --- a/extensions-builtin/LDSR/sd_hijack_ddpm_v1.py +++ b/extensions-builtin/LDSR/sd_hijack_ddpm_v1.py @@ -162,7 +162,6 @@ class DDPMV1(pl.LightningModule): lvlb_weights = 0.5 * np.sqrt(torch.Tensor(alphas_cumprod)) / (2. * 1 - torch.Tensor(alphas_cumprod)) else: raise NotImplementedError("mu not supported") - # TODO how to choose this term lvlb_weights[0] = lvlb_weights[1] self.register_buffer('lvlb_weights', lvlb_weights, persistent=False) assert not torch.isnan(self.lvlb_weights).all() @@ -872,7 +871,7 @@ class LatentDiffusionV1(DDPMV1): assert c is not None if self.cond_stage_trainable: c = self.get_learned_conditioning(c) - if self.shorten_cond_schedule: # TODO: drop this option + if self.shorten_cond_schedule: tc = self.cond_ids[t].to(self.device) c = self.q_sample(x_start=c, t=tc, noise=torch.randn_like(c.float())) return self.p_losses(x, c, t, *args, **kwargs) diff --git a/installer.py b/installer.py index 07de36730..795e81272 100644 --- a/installer.py +++ b/installer.py @@ -212,8 +212,10 @@ def git(arg: str, folder: str = None, ignore: bool = False): # switch to main branch as head can get detached def branch(folder): + if args.experimental: + return None if not os.path.exists(os.path.join(folder, '.git')): - return + return None b = git('branch', folder) if 'main' in b: b = 'main' diff --git a/javascript/black-orange.css b/javascript/black-orange.css index 09cd5bb0f..20e9f7ec6 100644 --- a/javascript/black-orange.css +++ b/javascript/black-orange.css @@ -18,28 +18,35 @@ --primary-900: #7c2d12; --primary-950: #6c2e12; } +.light, .dark { + --radius-lg: 2px; + --radius-sm: 1px; + --spacing-md: 5px; +} + html { font-size: var(--font-size); } body, button, input, select, textarea { font-family: var(--font);} button { font-size: 1.2rem; } img { background-color: var(--background-color); } input[type=range] { height: 18px; appearance: none; margin-top: 0; min-width: 160px; background-color: var(--background-color); width: 100%; background: transparent; } -input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: #50555C; border-radius: 2px; border: 0px solid #222222; } -input[type=range]::-moz-range-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: #50555C; border-radius: 2px; border: 0px solid #222222; } -input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: 2px; background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; } -input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: 2px; background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; } +input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: #50555C; border-radius: var(--radius-lg); border: 0px solid #222222; } +input[type=range]::-moz-range-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: #50555C; border-radius: var(--radius-lg); border: 0px solid #222222; } +input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: var(--radius-lg); background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; } +input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: var(--radius-lg); background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; } ::-webkit-scrollbar { width: 12px; } ::-webkit-scrollbar-track { background: #333333; } -::-webkit-scrollbar-thumb { background-color: var(--highlight-color); border-radius: 2px; border-width: 0; box-shadow: 2px 2px 3px #111111; } +::-webkit-scrollbar-thumb { background-color: var(--highlight-color); border-radius: var(--radius-lg); border-width: 0; box-shadow: 2px 2px 3px #111111; } div.form { border-width: 0; box-shadow: none; background: transparent; overflow: visible; gap: 0.5em; margin-bottom: 6px; } +div.compact { gap: 0.8em; } /* gradio style classes */ fieldset .gr-block.gr-box, label.block span { padding: 0; margin-top: -4px; } .border-2 { border-width: 0; } .border-b-2 { border-bottom-width: 2px; border-color: var(--highlight-color) !important; padding-bottom: 2px; margin-bottom: 8px; } .bg-white { color: lightyellow; background-color: var(--inactive-color); } -.gr-box { border-radius: 0 !important; background-color: #111111 !important; box-shadow: 2px 2px 3px #111111; border-width: 0; padding: 4px; margin: 12px 0px 12px 0px } +.gr-box { border-radius: var(--radius-sm) !important; background-color: #111111 !important; box-shadow: 2px 2px 3px #111111; border-width: 0; padding: 4px; margin: 12px 0px 12px 0px } .gr-button { font-weight: normal; box-shadow: 2px 2px 3px #111111; font-size: 0.8rem; min-width: 32px; min-height: 32px; padding: 3px; margin: 3px; } -.gr-check-radio { background-color: var(--inactive-color); border-width: 0; border-radius: 2px; box-shadow: 2px 2px 3px #111111; } +.gr-check-radio { background-color: var(--inactive-color); border-width: 0; border-radius: var(--radius-lg); box-shadow: 2px 2px 3px #111111; } .gr-check-radio:checked { background-color: var(--highlight-color); } .gr-compact { background-color: var(--background-color); } .gr-form { border-width: 0; } @@ -66,8 +73,8 @@ svg.feather.feather-image, .feather .feather-image { display: none } #tab_extensions table thead { background-color: var(--neutral-700); } /* automatic style classes */ -.progressDiv { border-radius: 0 !important; position: fixed; top: 44px; right: 26px; max-width: 262px; height: 48px; z-index: 99; box-shadow: var(--button-shadow); } -.progressDiv .progress { border-radius: 0 !important; background: var(--highlight-color); line-height: 3rem; height: 48px; } +.progressDiv { border-radius: var(--radius-sm) !important; position: fixed; top: 44px; right: 26px; max-width: 262px; height: 48px; z-index: 99; box-shadow: var(--button-shadow); } +.progressDiv .progress { border-radius: var(--radius-lg) !important; background: var(--highlight-color); line-height: 3rem; height: 48px; } .gallery-item { box-shadow: none !important; } .performance { color: #888; } .extra-networks { border-left: 2px solid var(--highlight-color) !important; padding-left: 4px; } @@ -77,15 +84,14 @@ svg.feather.feather-image, .feather .feather-image { display: none } #div.gradio-container { overflow-x: hidden; } #img2img_label_copy_to_img2img { font-weight: normal; } #txt2img_prompt, #txt2img_neg_prompt, #img2img_prompt, #img2img_neg_prompt { background-color: var(--background-color); box-shadow: 4px 4px 4px 0px #333333 !important; } -#txt2img_prompt > label > textarea, #txt2img_neg_prompt > label > textarea, #img2img_prompt > label > textarea, #img2img_neg_prompt > label > textarea { font-size: 1.2rem; } +#txt2img_prompt > label > textarea, #txt2img_neg_prompt > label > textarea, #img2img_prompt > label > textarea, #img2img_neg_prompt > label > textarea { font-size: 1.1rem; } #img2img_settings { min-width: calc(2 * var(--left-column)); max-width: calc(2 * var(--left-column)); background-color: #111111; padding-top: 16px; } #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; } #lightboxModal { background-color: rgba(20, 20, 20, 0.8) } #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-left: -20px; margin-top: -2px; height: 2.4em; } #quicksettings > div, #quicksettings > fieldset { min-width: 24em; max-width: 26em; line-height: 2em; } -#refresh_sd_model_checkpoint { height: 48px; margin-left: -14px; background: #333333; box-shadow: none; } #open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; } -#save-animation { border-radius: 0 !important; margin-bottom: 16px; background-color: #111111; } +#save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: #111111; } #script_list { padding: 4px; margin-top: 20px; margin-bottom: 20px; } #settings > div.flex-wrap { width: 15em; } #tab_extensions table { background-color: #222222; } @@ -222,15 +228,12 @@ svg.feather.feather-image, .feather .feather-image { display: none } --spacing-xxs: 1px; --spacing-xs: 2px; --spacing-sm: 4px; - --spacing-md: 6px; --spacing-lg: 8px; --spacing-xl: 10px; --spacing-xxl: 18px; --radius-xxs: 0; --radius-xs: 0; - --radius-sm: 0; --radius-md: 0; - --radius-lg: 0; --radius-xl: 0; --radius-xxl: 0; --text-xxs: 9px; diff --git a/modules/devices.py b/modules/devices.py index eedbb432b..96a49baa8 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -8,6 +8,8 @@ if sys.platform == "darwin": from modules import mac_specific # pylint: disable=ungrouped-imports cuda_ok = torch.cuda.is_available() +previous_oom = 0 + def has_mps() -> bool: if sys.platform != "darwin": @@ -15,6 +17,7 @@ def has_mps() -> bool: else: return mac_specific.has_mps + def extract_device_id(args, name): # pylint: disable=redefined-outer-name for x in range(len(args)): if name in args[x]: @@ -61,6 +64,18 @@ def get_device_for(task): def torch_gc(force=False): + mem = memstats.memory_stats() + gpu = mem.get('gpu', {}) + oom = gpu.get('oom', 0) + used = round(100 * gpu.get('used', 0) / gpu.get('total', 1)) + global previous_oom # pylint: disable=global-statement + if oom > previous_oom: + previous_oom = oom + shared.log.warning(f'GPU out-of-memory error: {mem}') + if used > 90: + shared.log.warning(f'GPU high memory utilization: {used}% {mem}') + force = True + if shared.opts.disable_gc and not force: return collected = gc.collect() diff --git a/modules/lora_diffusers.py b/modules/lora_diffusers.py index a6a5cbad7..e5779b3c6 100644 --- a/modules/lora_diffusers.py +++ b/modules/lora_diffusers.py @@ -1,7 +1,7 @@ import diffusers from modules import shared -lora_state = { # TODO this is ugly but diffusers +lora_state = { # TODO Lora state for Diffusers 'multiplier': 1.0, 'active': False, 'loaded': 0, diff --git a/modules/modelloader.py b/modules/modelloader.py index 6de1efe6a..a2d616b01 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -30,7 +30,10 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config shared.log.debug(f"Diffusers authentication: {token}") hf.login(token) pipeline_dir = DiffusionPipeline.download(hub_id, **download_config) - model_info_dict = hf.model_info(hub_id).cardData # TODO hfhub card-data? + try: + model_info_dict = hf.model_info(hub_id).cardData # TODO HF-Hub cardData invalid property + except Exception: + model_info_dict = None # some checkpoints need to be downloaded as "hidden" as they just serve as pre- or post-pipelines of other pipelines if model_info_dict is not None and "prior" in model_info_dict: download_dir = DiffusionPipeline.download(model_info_dict["prior"], **download_config) diff --git a/modules/models/diffusion/ddpm_edit.py b/modules/models/diffusion/ddpm_edit.py index 72e011d6f..ad067dd8b 100644 --- a/modules/models/diffusion/ddpm_edit.py +++ b/modules/models/diffusion/ddpm_edit.py @@ -174,7 +174,6 @@ class DDPM(pl.LightningModule): lvlb_weights = 0.5 * np.sqrt(torch.Tensor(alphas_cumprod)) / (2. * 1 - torch.Tensor(alphas_cumprod)) else: raise NotImplementedError("mu not supported") - # TODO how to choose this term lvlb_weights[0] = lvlb_weights[1] self.register_buffer('lvlb_weights', lvlb_weights, persistent=False) assert not torch.isnan(self.lvlb_weights).all() @@ -887,7 +886,7 @@ class LatentDiffusion(DDPM): assert c is not None if self.cond_stage_trainable: c = self.get_learned_conditioning(c) - if self.shorten_cond_schedule: # TODO: drop this option + if self.shorten_cond_schedule: tc = self.cond_ids[t].to(self.device) c = self.q_sample(x_start=c, t=tc, noise=torch.randn_like(c.float())) return self.p_losses(x, c, t, *args, **kwargs) @@ -1430,7 +1429,6 @@ class DiffusionWrapper(pl.LightningModule): class Layout2ImgDiffusion(LatentDiffusion): - # TODO: move all layout-specific hacks to this class def __init__(self, cond_stage_key, *args, **kwargs): assert cond_stage_key == 'coordinates_bbox', 'Layout2ImgDiffusion only for cond_stage_key="coordinates_bbox"' super().__init__(*args, cond_stage_key=cond_stage_key, **kwargs) diff --git a/modules/processing.py b/modules/processing.py index 9ebd20ede..4854a1c31 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -447,24 +447,23 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_su if uses_ensd: uses_ensd = sd_samplers_common.is_sampler_using_eta_noise_seed_delta(p) - generation_params = { "Steps": p.steps, "Sampler": p.sampler_name, "CFG scale": p.cfg_scale, "Image CFG scale": getattr(p, 'image_cfg_scale', None), "Seed": all_seeds[index], - "Face restoration": (opts.face_restoration_model if p.restore_faces else None), + "Face restoration": opts.face_restoration_model if p.restore_faces else None, "Size": f"{p.width}x{p.height}", "Model hash": getattr(p, 'sd_model_hash', None if not opts.add_model_hash_to_info or not shared.sd_model.sd_model_hash else shared.sd_model.sd_model_hash), - "Model": (None if not opts.add_model_name_to_info or not shared.sd_model.sd_checkpoint_info.model_name else shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', '')), - "VAE": (None if not opts.add_model_name_to_info or sd_vae.loaded_vae_file is None else os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0]), - "Variation seed": (None if p.subseed_strength == 0 else all_subseeds[index]), - "Variation seed strength": (None if p.subseed_strength == 0 else p.subseed_strength), - "Seed resize from": (None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}"), + "Model": None if not opts.add_model_name_to_info or not shared.sd_model.sd_checkpoint_info.model_name else shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', ''), + "VAE": None if not opts.add_model_name_to_info or sd_vae.loaded_vae_file is None else os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0], + "Variation seed": None if p.subseed_strength == 0 else all_subseeds[index], + "Variation seed strength": None if p.subseed_strength == 0 else p.subseed_strength, + "Seed resize from": None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}", "Denoising strength": getattr(p, 'denoising_strength', None), "Conditional mask weight": getattr(p, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) if p.is_using_inpainting_conditioning else None, - "Clip skip": p.clip_skip, + "Clip skip": p.clip_skip if p.clip_skip > 1 else None, "ENSD": opts.eta_noise_seed_delta if uses_ensd else None, "Init image hash": getattr(p, 'init_img_hash', None), "Version": git_commit, @@ -705,7 +704,9 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: # TODO(PVP): change out to latents once possible with `diffusers` task_specific_kwargs = {"image": p.init_images[0], "mask_image": p.image_mask, "strength": p.denoising_strength} - def diffusers_callback(step: int, _timestep: int, latents: torch.FloatTensor): # TODO simplified callback for now + # TODO Diffusers limited callbacks + # TODO Diffusers processing is not using p.sample so second pass is ignored + def diffusers_callback(step: int, _timestep: int, latents: torch.FloatTensor): shared.state.sampling_step = step shared.state.sampling_steps = p.steps shared.state.current_latent = latents @@ -728,9 +729,9 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: # shared.sd_model.to('cpu') # devices.torch_gc(force=True) - if shared.sd_refiner is not None: # shared.sd_refiner.to(devices.device) + devices.torch_gc() init_image = output.images[0] output = shared.sd_refiner( # pylint: disable=not-callable prompt=prompts, diff --git a/modules/sd_models.py b/modules/sd_models.py index 505d79101..dcc7d4df5 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -59,6 +59,8 @@ class CheckpointInfo: self.sha256 = hashes.sha256_from_cache(self.filename, f"checkpoint/{name}") self.path = abspath self.type = abspath.split('.')[-1].lower() + self.name_for_extra = os.path.splitext(os.path.basename(filename))[0] + self.model_name = os.path.splitext(name.replace("/", "_").replace("\\", "_"))[0] else: # maybe a diffuser repo = [r for r in modelloader.diffuser_repos if filename == r['filename']] if len(repo) == 0: @@ -70,16 +72,13 @@ class CheckpointInfo: self.sha256 = repo[0]['hash'] self.path = repo[0]['path'] self.type = 'diffusers' - + self.name_for_extra = repo[0]['name'] + self.model_name = repo[0]['name'] if os.path.isfile(repo[0]['model_info']): file_path = repo[0]['model_info'] with open(file_path, "r", encoding="utf-8") as json_file: self.model_info = json.load(json_file) - else: - self.model_info = None - self.name_for_extra = os.path.splitext(os.path.basename(filename))[0] - self.model_name = os.path.splitext(name.replace("/", "_").replace("\\", "_"))[0] self.shorthash = self.sha256[0:10] if self.sha256 else None self.title = self.name if self.shorthash is None else f'{self.name} [{self.shorthash}]' self.ids = [self.hash, self.model_name, self.title, self.name, f'{self.name} [{self.hash}]'] + ([self.shorthash, self.sha256, f'{self.name} [{self.shorthash}]'] if self.shorthash else []) @@ -161,7 +160,7 @@ def list_models(): model_list = modelloader.load_models(model_path=model_path, model_url=model_url, command_path=shared.opts.ckpt_dir, ext_filter=[".ckpt", ".safetensors"], download_name="v1-5-pruned-emaonly.safetensors", ext_blacklist=[".vae.ckpt", ".vae.safetensors"]) else: default_model_id = "runwayml/stable-diffusion-v1-5" - modelloader.download_diffusers_model(default_model_id, os.path.join(models_path, 'Diffusers')) + modelloader.download_diffusers_model(default_model_id, shared.opts.diffusers_dir) model_list = modelloader.load_diffusers_models(model_path=os.path.join(models_path, 'Diffusers'), command_path=shared.opts.diffusers_dir) for filename in sorted(model_list, key=str.lower): @@ -665,6 +664,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No else: shared.log.error(f'Diffusers cannot load safetensor model: {checkpoint_info.path} {shared.opts.diffusers_pipeline}') return + if sd_model is not None: + shared.log.debug(f'Diffusers pipeline: {type(sd_model)}') # pylint: disable=protected-access except Exception as e: shared.log.error(f'Diffusers failed loading model using pipeline: {checkpoint_info.path} {shared.opts.diffusers_pipeline} {e}') return diff --git a/modules/ui_models.py b/modules/ui_models.py index 325eca125..592fb2e08 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -219,5 +219,5 @@ def create_ui(): hf_results.select(fn=hf_select, inputs=[hf_results], outputs=[hf_selected]) hf_download_model_btn.click(fn=hf_download_model, inputs=[hf_selected, hf_token], outputs=[models_outcome]) - with gr.Tab(label="CivitAI"): - pass + # with gr.Tab(label="CivitAI"): + # pass diff --git a/requirements.txt b/requirements.txt index dd1ada5a4..7f93f9fa3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -59,7 +59,7 @@ pandas==1.5.3 protobuf==3.20.3 pytorch_lightning==1.9.4 transformers==4.30.2 -timm==0.6.13 tomesd==0.1.3 urllib3==1.26.15 Pillow==9.5.0 +timm==0.6.13 diff --git a/wiki b/wiki index 503fa982c..e5da783be 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 503fa982cba22a33f3601f43fd255883167e421c +Subproject commit e5da783bef6cacdb669c482d726e23a8d20d86d8 From a79b8c86c260b4dbb7a71dff241579531108845a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 8 Jul 2023 12:20:37 -0400 Subject: [PATCH 41/42] cleanup before merge --- .gitmodules | 28 ++++++++++++++-------------- TODO.md | 2 +- modules/api/models.py | 3 +-- modules/sd_models.py | 4 ++-- requirements.txt | 2 +- webui.py | 1 + 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.gitmodules b/.gitmodules index 6b435fe50..4212e20fd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,29 +17,29 @@ url = https://github.com/kohya-ss/sd-scripts ignore = dirty [submodule "extensions-builtin/clip-interrogator-ext"] - path = extensions-builtin/clip-interrogator-ext - url = https://github.com/Dahvikiin/clip-interrogator-ext.git + path = extensions-builtin/clip-interrogator-ext + url = https://github.com/Dahvikiin/clip-interrogator-ext.git ignore = dirty [submodule "extensions-builtin/sd-webui-controlnet"] - path = extensions-builtin/sd-webui-controlnet - url = https://github.com/Mikubill/sd-webui-controlnet + path = extensions-builtin/sd-webui-controlnet + url = https://github.com/Mikubill/sd-webui-controlnet ignore = dirty [submodule "modules/lycoris"] - path = modules/lycoris - url = https://github.com/KohakuBlueleaf/LyCORIS + path = modules/lycoris + url = https://github.com/KohakuBlueleaf/LyCORIS ignore = dirty [submodule "extensions-builtin/stable-diffusion-webui-rembg"] - path = extensions-builtin/stable-diffusion-webui-rembg - url = https://github.com/AUTOMATIC1111/stable-diffusion-webui-rembg + path = extensions-builtin/stable-diffusion-webui-rembg + url = https://github.com/AUTOMATIC1111/stable-diffusion-webui-rembg ignore = dirty [submodule "extensions-builtin/a1111-sd-webui-lycoris"] - path = extensions-builtin/a1111-sd-webui-lycoris - url = https://github.com/KohakuBlueleaf/a1111-sd-webui-lycoris + path = extensions-builtin/a1111-sd-webui-lycoris + url = https://github.com/KohakuBlueleaf/a1111-sd-webui-lycoris ignore = dirty [submodule "extensions-builtin/multidiffusion-upscaler-for-automatic1111"] - path = extensions-builtin/multidiffusion-upscaler-for-automatic1111 - url = https://github.com/pkuliyi2015/multidiffusion-upscaler-for-automatic1111 + path = extensions-builtin/multidiffusion-upscaler-for-automatic1111 + url = https://github.com/pkuliyi2015/multidiffusion-upscaler-for-automatic1111 ignore = dirty [submodule "extensions-builtin/sd-webui-agent-scheduler"] - path = extensions-builtin/sd-webui-agent-scheduler - url = https://github.com/ArtVentureX/sd-webui-agent-scheduler + path = extensions-builtin/sd-webui-agent-scheduler + url = https://github.com/ArtVentureX/sd-webui-agent-scheduler diff --git a/TODO.md b/TODO.md index e2f68a7de..9ee756690 100644 --- a/TODO.md +++ b/TODO.md @@ -60,6 +60,6 @@ Tech that can be integrated as part of the core workflow... - git-rebasin - additional upscalers - new image browser -- `fp8` - update `transformers` - `git submodule set-url extensions-builtin/clip-interrogator-ext https://github.com/Dahvikiin/clip-interrogator-ext.git` +- upate `gradio` diff --git a/modules/api/models.py b/modules/api/models.py index 1b3399f18..704ebfe92 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -1,7 +1,6 @@ import inspect from typing import Any, Optional, Dict, List from pydantic import BaseModel, Field, create_model # pylint: disable=no-name-in-module -from typing_extensions import Literal from inflection import underscore from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img import modules.shared as shared @@ -133,7 +132,7 @@ class ImageToImageResponse(BaseModel): info: str class ExtrasBaseRequest(BaseModel): - resize_mode: Literal[0, 1] = Field(default=0, title="Resize Mode", description="Sets the resize mode: 0 to upscale by upscaling_resize amount, 1 to upscale up to upscaling_resize_h x upscaling_resize_w.") + resize_mode: float = Field(default=0, title="Resize Mode", description="Sets the resize mode: 0 to upscale by upscaling_resize amount, 1 to upscale up to upscaling_resize_h x upscaling_resize_w.") show_extras_results: bool = Field(default=True, title="Show results", description="Should the backend return the generated image?") gfpgan_visibility: float = Field(default=0, title="GFPGAN Visibility", ge=0, le=1, allow_inf_nan=False, description="Sets the visibility of GFPGAN, values should be between 0 and 1.") codeformer_visibility: float = Field(default=0, title="CodeFormer Visibility", ge=0, le=1, allow_inf_nan=False, description="Sets the visibility of CodeFormer, values should be between 0 and 1.") diff --git a/modules/sd_models.py b/modules/sd_models.py index dcc7d4df5..d1a794a62 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -583,10 +583,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No shared.opts.data['sd_model_checkpoint'] = "runwayml/stable-diffusion-v1-5" if op == 'model' or op == 'dict': - if model_data.sd_model is not None and (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model + if model_data.sd_model is not None and checkpoint_info is not None (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model return else: - if model_data.sd_refiner is not None and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model + if model_data.sd_refiner is not None and checkpoint_info is not None and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model return shared.log.debug(f'Diffusers load config: {diffusers_load_config}') diff --git a/requirements.txt b/requirements.txt index 7f93f9fa3..f47577cd7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,7 +43,6 @@ scikit-image basicsr compel antlr4-python3-runtime==4.9.3 -typing-extensions==4.6.3 pydantic==1.10.9 requests==2.31.0 tqdm==4.65.0 @@ -63,3 +62,4 @@ tomesd==0.1.3 urllib3==1.26.15 Pillow==9.5.0 timm==0.6.13 +typing-extensions==4.6.3 diff --git a/webui.py b/webui.py index 1b8da00c7..3b6657dbe 100644 --- a/webui.py +++ b/webui.py @@ -32,6 +32,7 @@ warnings.filterwarnings(action="ignore", category=UserWarning, module="torchvisi startup_timer.record("torch") errors.log.debug('Loading Gradio') +import typing_extensions # pylint: disable=W0611,C0411 from fastapi import FastAPI # pylint: disable=W0611,C0411 import gradio # pylint: disable=W0611,C0411 startup_timer.record("gradio") From 3e61907bfe215c0797ed14154be9c42c6e6c7ee7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 8 Jul 2023 13:17:12 -0400 Subject: [PATCH 42/42] minor fixes --- TODO.md | 1 + javascript/extraNetworks.js | 2 +- modules/sd_models.py | 4 ++-- requirements.txt | 4 ++-- webui.py | 1 - 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/TODO.md b/TODO.md index 9ee756690..8d5b1a975 100644 --- a/TODO.md +++ b/TODO.md @@ -63,3 +63,4 @@ Tech that can be integrated as part of the core workflow... - update `transformers` - `git submodule set-url extensions-builtin/clip-interrogator-ext https://github.com/Dahvikiin/clip-interrogator-ext.git` - upate `gradio` +- extra network refresh breaks if new extra network type found diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 5f02ed586..ad0babd6a 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -27,7 +27,7 @@ function setupExtraNetworksForTab(tabname) { intersectionObserver = new IntersectionObserver((entries) => { if (!en) return - for (el of Array.from(gradioApp().querySelectorAll('.extra-network-cards'))) el.style.height = window.opts.extra_networks_height + 'vh'; + for (el of Array.from(gradioApp().querySelectorAll('.extra-networks-page'))) el.style.height = window.opts.extra_networks_height + 'vh'; if (entries[0].intersectionRatio > 0) { if (window.opts.extra_networks_card_cover === 'cover') { en.style.transition = ''; diff --git a/modules/sd_models.py b/modules/sd_models.py index d1a794a62..cc6c8d3d5 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -583,10 +583,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No shared.opts.data['sd_model_checkpoint'] = "runwayml/stable-diffusion-v1-5" if op == 'model' or op == 'dict': - if model_data.sd_model is not None and checkpoint_info is not None (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model + if (model_data.sd_model is not None) and (checkpoint_info is not None) and (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model return else: - if model_data.sd_refiner is not None and checkpoint_info is not None and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model + if (model_data.sd_refiner is not None) and (checkpoint_info is not None) and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model return shared.log.debug(f'Diffusers load config: {diffusers_load_config}') diff --git a/requirements.txt b/requirements.txt index f47577cd7..59cc27e6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,8 +42,9 @@ yapf scikit-image basicsr compel +typing-extensions==4.7.1 antlr4-python3-runtime==4.9.3 -pydantic==1.10.9 +pydantic==1.10.11 requests==2.31.0 tqdm==4.65.0 accelerate==0.20.3 @@ -62,4 +63,3 @@ tomesd==0.1.3 urllib3==1.26.15 Pillow==9.5.0 timm==0.6.13 -typing-extensions==4.6.3 diff --git a/webui.py b/webui.py index 3b6657dbe..1b8da00c7 100644 --- a/webui.py +++ b/webui.py @@ -32,7 +32,6 @@ warnings.filterwarnings(action="ignore", category=UserWarning, module="torchvisi startup_timer.record("torch") errors.log.debug('Loading Gradio') -import typing_extensions # pylint: disable=W0611,C0411 from fastapi import FastAPI # pylint: disable=W0611,C0411 import gradio # pylint: disable=W0611,C0411 startup_timer.record("gradio")