diff --git a/.gitmodules b/.gitmodules index 93009f907..4212e20fd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,29 +17,29 @@ url = https://github.com/kohya-ss/sd-scripts ignore = dirty [submodule "extensions-builtin/clip-interrogator-ext"] - path = extensions-builtin/clip-interrogator-ext - url = https://github.com/pharmapsychotic/clip-interrogator-ext.git + path = extensions-builtin/clip-interrogator-ext + url = https://github.com/Dahvikiin/clip-interrogator-ext.git ignore = dirty [submodule "extensions-builtin/sd-webui-controlnet"] - path = extensions-builtin/sd-webui-controlnet - url = https://github.com/Mikubill/sd-webui-controlnet + path = extensions-builtin/sd-webui-controlnet + url = https://github.com/Mikubill/sd-webui-controlnet ignore = dirty [submodule "modules/lycoris"] - path = modules/lycoris - url = https://github.com/KohakuBlueleaf/LyCORIS + path = modules/lycoris + url = https://github.com/KohakuBlueleaf/LyCORIS ignore = dirty [submodule "extensions-builtin/stable-diffusion-webui-rembg"] - path = extensions-builtin/stable-diffusion-webui-rembg - url = https://github.com/AUTOMATIC1111/stable-diffusion-webui-rembg + path = extensions-builtin/stable-diffusion-webui-rembg + url = https://github.com/AUTOMATIC1111/stable-diffusion-webui-rembg ignore = dirty [submodule "extensions-builtin/a1111-sd-webui-lycoris"] - path = extensions-builtin/a1111-sd-webui-lycoris - url = https://github.com/KohakuBlueleaf/a1111-sd-webui-lycoris + path = extensions-builtin/a1111-sd-webui-lycoris + url = https://github.com/KohakuBlueleaf/a1111-sd-webui-lycoris ignore = dirty [submodule "extensions-builtin/multidiffusion-upscaler-for-automatic1111"] - path = extensions-builtin/multidiffusion-upscaler-for-automatic1111 - url = https://github.com/pkuliyi2015/multidiffusion-upscaler-for-automatic1111 + path = extensions-builtin/multidiffusion-upscaler-for-automatic1111 + url = https://github.com/pkuliyi2015/multidiffusion-upscaler-for-automatic1111 ignore = dirty [submodule "extensions-builtin/sd-webui-agent-scheduler"] - path = extensions-builtin/sd-webui-agent-scheduler - url = https://github.com/ArtVentureX/sd-webui-agent-scheduler + path = extensions-builtin/sd-webui-agent-scheduler + url = https://github.com/ArtVentureX/sd-webui-agent-scheduler diff --git a/CHANGELOG.md b/CHANGELOG.md index e60a29277..dfdf9f26d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Change Log for SD.Next +## Update for 07/08/2023 + +- add pan & zoom controls (touch and mouse) to image viewer (lightbox) +- cache extra networks between tabs + this should result in neat 2x speedup on building extra networks +- add settings -> extra networks -> do not automatically build extra network pages + speeds up app start if you have a lot of extra networks and you want to build them manually when needed +- extra network ui tweaks +- merge experimental diffusers support + covered in details in a separate post + ## Update for 07/01/2023 Small quality-of-life updates and bugfixes: diff --git a/README.md b/README.md index c4e55125d..012a1fc62 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,19 @@ Below is partial list of all available parameters, run `webui --help` for the fu ## Notes +### **Extensions** + +SD.Next comes with several extensions pre-installed: + +- [Dyanmic Thresholding](https://github.com/mcmonkeyprojects/sd-dynamic-thresholding) +- [ControlNet](https://github.com/Mikubill/sd-webui-controlnet) +- [Agent Scheduler](https://github.com/ArtVentureX/sd-webui-agent-scheduler) +- [Multi-Diffusion Tiled Diffusion and VAE](https://github.com/pkuliyi2015/multidiffusion-upscaler-for-automatic1111) +- [LyCORIS](https://github.com/KohakuBlueleaf/a1111-sd-webui-lycoris) +- [Image Browser](https://github.com/AlUlkesh/stable-diffusion-webui-images-browser) +- [CLiP Interrogator](https://github.com/pharmapsychotic/clip-interrogator-ext) +- [Rembg Background Removal](https://github.com/AUTOMATIC1111/stable-diffusion-webui-rembg) + ### **Collab** - To avoid having this repo rely just on me, I'd love to have additional maintainers with full admin rights. If you're interested, ping me! diff --git a/TODO.md b/TODO.md index 19ce7d49f..8d5b1a975 100644 --- a/TODO.md +++ b/TODO.md @@ -26,7 +26,8 @@ Stuff to be investigated... Pick & merge PRs from main repo... -- list: +- up-to-date with: df004be +- current todo list: ## Integration @@ -41,13 +42,13 @@ Tech that can be integrated as part of the core workflow... - [DataComp CLiP](https://github.com/mlfoundations/open_clip/blob/main/docs/datacomp_models.md) - [ClipSeg](https://github.com/timojl/clipseg) - [DragGAN](https://github.com/XingangPan/DragGAN) +- [LamaCleaner]([Title](https://github.com/Sanster/lama-cleaner)) - `TensorRT` ## Random - Bunch of stuff: - -- shared.info - docker - port `p.all_hr_prompts` - test `lyco_patch_lora` @@ -56,6 +57,10 @@ Tech that can be integrated as part of the core workflow... - image watermark - image `imagehash` phash and hdash - custom exif tags -- replace lightbox with iv-viewer - git-rebasin - additional upscalers +- new image browser +- update `transformers` +- `git submodule set-url extensions-builtin/clip-interrogator-ext https://github.com/Dahvikiin/clip-interrogator-ext.git` +- upate `gradio` +- extra network refresh breaks if new extra network type found diff --git a/cli/hf-search.py b/cli/hf-search.py index bbb4a4f0f..ac97b6c26 100755 --- a/cli/hf-search.py +++ b/cli/hf-search.py @@ -11,9 +11,8 @@ if __name__ == "__main__": model_filter = hf.ModelFilter( model_name=keyword, task='text-to-image', - tags='stable-diffusion', - library=['diffusers', 'stable-diffusion'], + library=['diffusers'], ) res = hf_api.list_models(filter=model_filter, full=True, limit=50, sort="downloads", direction=-1) - models = [{ 'name': m.modelId, 'downloads': m.downloads, 'mtime': m.lastModified, 'url': f'https://huggingface.co/{m.modelId}' } for m in res] - print('Online', models) + models = [{ 'name': m.modelId, 'downloads': m.downloads, 'mtime': m.lastModified, 'url': f'https://huggingface.co/{m.modelId}', 'pipeline': m.pipeline_tag, 'tags': m.tags } for m in res] + print(models) diff --git a/cli/sdapi.py b/cli/sdapi.py index 3d2d99ab8..3b010e663 100755 --- a/cli/sdapi.py +++ b/cli/sdapi.py @@ -177,7 +177,7 @@ def get_log(): def get_info(): import time t0 = time.time() - res = getsync('/sdapi/v1/system-info/status') + res = getsync('/sdapi/v1/system-info/status?full=true&refresh=true') t1 = time.time() print({ 'duration': 1000 * round(t1-t0, 3), **res }) return res diff --git a/cli/train.py b/cli/train.py index cf27fbcc8..97268dd54 100755 --- a/cli/train.py +++ b/cli/train.py @@ -265,6 +265,7 @@ def prepare_options(): options.lora.in_json = None if args.type == 'dreambooth': log.info('train using dreambooth style training') + options.lora.vae_batch_size = args.batch options.lora.in_json = None if args.type == 'lora': log.info('train using lora style training') @@ -378,14 +379,18 @@ def check_versions(): log.info('experimental mode enabled') return log.info('checking accelerate') + error = False import accelerate if accelerate.__version__ != '0.19.0': - log.error(f'invalid accelerate version: required=0.19.0 found={accelerate.__version__}') - exit(1) + log.error(f'invalid accelerate version: accelerate=0.19.0 found={accelerate.__version__}') + error = True log.info('checking diffusers') import diffusers if diffusers.__version__ != '0.10.2': - log.error(f'invalid diffusers version: required=0.10.2 found={diffusers.__version__}') + log.error(f'invalid diffusers version: diffusers=0.10.2 found={diffusers.__version__}') + error = True + if error: + log.info('> pip install accelerate==0.19.0 diffusers==0.10.2') exit(1) diff --git a/extensions-builtin/LDSR/ldsr_model_arch.py b/extensions-builtin/LDSR/ldsr_model_arch.py index f0a90c657..5299f2c29 100644 --- a/extensions-builtin/LDSR/ldsr_model_arch.py +++ b/extensions-builtin/LDSR/ldsr_model_arch.py @@ -23,10 +23,10 @@ class LDSR: global cached_ldsr_model if shared.opts.ldsr_cached and cached_ldsr_model is not None: - print("Loading model from cache") + shared.log.info("LDSR Loading model from cache") model: torch.nn.Module = cached_ldsr_model else: - print(f"Loading model from {self.modelPath}") + shared.log.info(f"LDSR Loading model from {self.modelPath}") _, extension = os.path.splitext(self.modelPath) if extension.lower() == ".safetensors": pl_sd = safetensors.torch.load_file(self.modelPath, device="cpu") @@ -126,11 +126,10 @@ class LDSR: height_downsampled_pre = int(np.ceil(hd)) if down_sample_rate != 1: - print( - f'Downsampling from [{width_og}, {height_og}] to [{width_downsampled_pre}, {height_downsampled_pre}]') + shared.log.info(f'LDSR Downsampling from [{width_og}, {height_og}] to [{width_downsampled_pre}, {height_downsampled_pre}]') im_og = im_og.resize((width_downsampled_pre, height_downsampled_pre), Image.LANCZOS) else: - print(f"Down sample rate is 1 from {target_scale} / 4 (Not downsampling)") + shared.log.info(f"LDSR Downsample rate is 1 from {target_scale} / 4 (Not downsampling)") # pad width and height to multiples of 64, pads with the edge values of image to avoid artifacts pad_w, pad_h = np.max(((2, 2), np.ceil(np.array(im_og.size) / 64).astype(int)), axis=0) * 64 - im_og.size @@ -183,7 +182,7 @@ def convsample_ddim(model, cond, steps, shape, eta=1.0, callback=None, normals_s ddim = DDIMSampler(model) bs = shape[0] shape = shape[1:] - print(f"Sampling with eta = {eta}; steps: {steps}") + shared.log.info(f"LDSR Sampling with eta = {eta}; steps: {steps}") samples, intermediates = ddim.sample(steps, batch_size=bs, shape=shape, conditioning=cond, callback=callback, normals_sequence=normals_sequence, quantize_x0=quantize_x0, eta=eta, mask=mask, x0=x0, temperature=temperature, verbose=False, @@ -206,7 +205,7 @@ def make_convolutional_sample(batch, model, custom_steps=None, eta=1.0, quantize if custom_shape is not None: z = torch.randn(custom_shape) - print(f"Generating {custom_shape[0]} samples of shape {custom_shape[1:]}") + shared.log.info(f"LDSR Generating {custom_shape[0]} samples of shape {custom_shape[1:]}") z0 = None diff --git a/extensions-builtin/LDSR/sd_hijack_ddpm_v1.py b/extensions-builtin/LDSR/sd_hijack_ddpm_v1.py index 631a08ef0..053be8290 100644 --- a/extensions-builtin/LDSR/sd_hijack_ddpm_v1.py +++ b/extensions-builtin/LDSR/sd_hijack_ddpm_v1.py @@ -162,7 +162,6 @@ class DDPMV1(pl.LightningModule): lvlb_weights = 0.5 * np.sqrt(torch.Tensor(alphas_cumprod)) / (2. * 1 - torch.Tensor(alphas_cumprod)) else: raise NotImplementedError("mu not supported") - # TODO how to choose this term lvlb_weights[0] = lvlb_weights[1] self.register_buffer('lvlb_weights', lvlb_weights, persistent=False) assert not torch.isnan(self.lvlb_weights).all() @@ -872,7 +871,7 @@ class LatentDiffusionV1(DDPMV1): assert c is not None if self.cond_stage_trainable: c = self.get_learned_conditioning(c) - if self.shorten_cond_schedule: # TODO: drop this option + if self.shorten_cond_schedule: tc = self.cond_ids[t].to(self.device) c = self.q_sample(x_start=c, t=tc, noise=torch.randn_like(c.float())) return self.p_losses(x, c, t, *args, **kwargs) @@ -958,7 +957,7 @@ class LatentDiffusionV1(DDPMV1): cond_list = [{'c_crossattn': [e]} for e in adapted_cond] else: - cond_list = [cond for i in range(z.shape[-1])] # Todo make this more efficient + cond_list = [cond for i in range(z.shape[-1])] # apply model by loop over crops output_list = [self.model(z_list[i], t, **cond_list[i]) for i in range(z.shape[-1])] diff --git a/extensions-builtin/Lora/extra_networks_lora.py b/extensions-builtin/Lora/extra_networks_lora.py index b5fea4d2e..bee0477ed 100644 --- a/extensions-builtin/Lora/extra_networks_lora.py +++ b/extensions-builtin/Lora/extra_networks_lora.py @@ -1,5 +1,5 @@ -from modules import extra_networks, shared import lora +from modules import extra_networks, shared class ExtraNetworkLora(extra_networks.ExtraNetwork): diff --git a/extensions-builtin/Lora/lora.py b/extensions-builtin/Lora/lora.py index 5273c9f82..088be31c1 100644 --- a/extensions-builtin/Lora/lora.py +++ b/extensions-builtin/Lora/lora.py @@ -1,8 +1,7 @@ import os import re -import torch from typing import Union - +import torch from modules import shared, devices, sd_models, errors, scripts, sd_hijack, hashes metadata_tags_order = {"ss_sd_model_name": 1, "ss_resolution": 2, "ss_clip_skip": 3, "ss_num_train_images": 10, "ss_tag_frequency": 20} @@ -127,7 +126,6 @@ class LoraModule: self.multiplier = 1.0 self.modules = {} self.mtime = None - self.mentioned_name = None """the text that was used to add lora to prompt - can be either name or an alias""" @@ -154,6 +152,13 @@ def assign_lora_names_to_compvis_modules(sd_model): sd_model.lora_layer_mapping = lora_layer_mapping +def load_diffuser_lora(name, lora_on_disk, multiplier): + lora = LoraModule(name, lora_on_disk) + lora.mtime = os.path.getmtime(lora_on_disk.filename) + from modules.lora_diffusers import load_diffusers_lora + load_diffusers_lora(name, lora_on_disk, multiplier) + return lora + def load_lora(name, lora_on_disk): lora = LoraModule(name, lora_on_disk) @@ -205,7 +210,6 @@ def load_lora(name, lora_on_disk): else: print(f'Lora layer {key_diffusers} matched a layer with unsupported type: {type(sd_module).__name__}') continue - raise AssertionError(f"Lora layer {key_diffusers} matched a layer with unsupported type: {type(sd_module).__name__}") with torch.no_grad(): module.weight.copy_(weight) @@ -243,14 +247,17 @@ def load_loras(names, multipliers=None): failed_to_load_loras = [] for i, name in enumerate(names): - lora = already_loaded.get(name, None) + lora = already_loaded.get(name, None) if shared.backend == shared.Backend.ORIGINAL else None lora_on_disk = loras_on_disk[i] if lora_on_disk is not None: if lora is None or os.path.getmtime(lora_on_disk.filename) > lora.mtime: try: - lora = load_lora(name, lora_on_disk) + if shared.backend == shared.Backend.DIFFUSERS: + lora = load_diffuser_lora(name, lora_on_disk, multipliers[i] if multipliers else 1.0) + else: + lora = load_lora(name, lora_on_disk) except Exception as e: errors.display(e, f"loading Lora {lora_on_disk.filename}") continue diff --git a/extensions-builtin/clip-interrogator-ext b/extensions-builtin/clip-interrogator-ext index c0bf90052..6e31272e1 160000 --- a/extensions-builtin/clip-interrogator-ext +++ b/extensions-builtin/clip-interrogator-ext @@ -1 +1 @@ -Subproject commit c0bf90052a14a104b2dbfd9b7e7818aae5ca5ae1 +Subproject commit 6e31272e14308b4918f9785b1dda7cc1149e8838 diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index b30e32455..9433a15b8 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit b30e324552517e68012f2487cf7b0be43616a4cc +Subproject commit 9433a15b8965f9b0aa6a93cd44992f29de3f3e79 diff --git a/extensions-builtin/sd-webui-agent-scheduler b/extensions-builtin/sd-webui-agent-scheduler index 8198489fd..75ba093d4 160000 --- a/extensions-builtin/sd-webui-agent-scheduler +++ b/extensions-builtin/sd-webui-agent-scheduler @@ -1 +1 @@ -Subproject commit 8198489fd42883d3af119e219c320ba4853802c8 +Subproject commit 75ba093d46b37f2d01fa4207680a4fa6417b01c3 diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 2785cbe61..dd766de86 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 2785cbe61a61c137d4e980752771f6329b03612a +Subproject commit dd766de8629ee6035a734217e08c26cd1b08b2ab diff --git a/html/locale_en.json b/html/locale_en.json index 3e69cd207..3f5617ee4 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -146,7 +146,7 @@ {"id":"","label":"System Paths","localized":"","hint":""}, {"id":"","label":"Image Options","localized":"","hint":""}, {"id":"","label":"Image Processing","localized":"","hint":""}, - {"id":"","label":"Output Paths","localized":"","hint":""}, + {"id":"","label":"Image Paths","localized":"","hint":""}, {"id":"","label":"User Interface","localized":"","hint":""}, {"id":"","label":"Live Previews","localized":"","hint":""}, {"id":"","label":"Sampler Settings","localized":"","hint":""}, @@ -524,9 +524,9 @@ ], "scripts": [ {"id":"","label":"Script","localized":"","hint":""}, - {"id":"","label":"Swap X/Y axes","localized":"","hint":""}, - {"id":"","label":"Swap Y/Z axes","localized":"","hint":""}, - {"id":"","label":"Swap X/Z axes","localized":"","hint":""}, + {"id":"","label":"Swap X/Y","localized":"","hint":""}, + {"id":"","label":"Swap Y/Z","localized":"","hint":""}, + {"id":"","label":"Swap X/Z","localized":"","hint":""}, {"id":"","label":"Resize to","localized":"","hint":""}, {"id":"","label":"Resize by","localized":"","hint":""}, {"id":"","label":"Use via API","localized":"","hint":""}, diff --git a/installer.py b/installer.py index c01b88fe0..41396c266 100644 --- a/installer.py +++ b/installer.py @@ -105,6 +105,7 @@ def setup_logging(): logging.getLogger("httpx").setLevel(logging.ERROR) logging.getLogger("ControlNet").handlers = log.handlers logging.getLogger("lycoris").handlers = log.handlers + # logging.getLogger("DeepSpeed").handlers = log.handlers def print_profile(profile: cProfile.Profile, msg: str): @@ -199,6 +200,8 @@ def git(arg: str, folder: str = None, ignore: bool = False): txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore") txt = txt.strip() if result.returncode != 0 and not ignore: + if "couldn't find remote ref" in txt: # not a git repo + return txt global errors # pylint: disable=global-statement errors += 1 log.error(f'Error running git: {folder} / {arg}') @@ -207,25 +210,34 @@ def git(arg: str, folder: str = None, ignore: bool = False): log.debug(f'Git output: {txt}') return txt - -# update switch to main branch as head can get detached and update repository -def update(folder): +# switch to main branch as head can get detached +def branch(folder): + if args.experimental: + return None if not os.path.exists(os.path.join(folder, '.git')): - return - branch = git('branch', folder) - if 'main' in branch: - branch = 'main' - elif 'master' in branch: - branch = 'master' + return None + b = git('branch', folder) + if 'main' in b: + b = 'main' + elif 'master' in b: + b = 'master' else: - branch = branch.split('\n')[0].replace('*', '').strip() - # log.debug(f'Setting branch: {folder} / {branch}') - git(f'checkout {branch}', folder) + b = b.split('\n')[0].replace('*', '').strip() + log.debug(f'Submodule: {folder} / {b}') + git(f'checkout {b}', folder, ignore=True) + return b + + +# update git repository +def update(folder, current_branch = False): + if current_branch: + git('pull --autostash --rebase --force', folder) + return + b = branch(folder) if branch is None: git('pull --autostash --rebase --force', folder) else: - git(f'pull origin {branch} --autostash --rebase --force', folder) - # branch = git('branch', folder) + git(f'pull origin {b} --autostash --rebase --force', folder) # clone git repository @@ -283,6 +295,7 @@ def check_torch(): log.debug(f'Torch overrides: cuda={args.use_cuda} rocm={args.use_rocm} ipex={args.use_ipex} diml={args.use_directml}') log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml}') torch_command = os.environ.get('TORCH_COMMAND', '') + xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') if torch_command != '': pass elif allow_cuda and (shutil.which('nvidia-smi') is not None or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))): @@ -301,7 +314,6 @@ def check_torch(): if shutil.which('sycl-ls') is None: log.error('Intel OneAPI Toolkit is not activated! Start the WebUI with --use-ipex or activate OneAPI manually') torch_command = os.environ.get('TORCH_COMMAND', 'torch==1.13.0a0 torchvision==0.14.1a0 intel_extension_for_pytorch==1.13.120+xpu -f https://developer.intel.com/ipex-whl-stable-xpu') - xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') else: machine = platform.machine() if sys.platform == 'darwin': @@ -309,13 +321,11 @@ def check_torch(): elif allow_directml and args.use_directml and ('arm' not in machine and 'aarch' not in machine): log.info('Using DirectML Backend') torch_command = os.environ.get('TORCH_COMMAND', 'torch-directml') - xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') if 'torch' in torch_command and not args.version: install(torch_command, 'torch torchvision') else: log.info('Using CPU-only Torch') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') - xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') if 'torch' in torch_command and not args.version: install(torch_command, 'torch torchvision') if args.skip_torch: @@ -405,6 +415,8 @@ def install_packages(): # install(openclip_package, 'open-clip-torch') clip_package = os.environ.get('CLIP_PACKAGE', "git+https://github.com/openai/CLIP.git") install(clip_package, 'clip') + invisiblewatermark_package = os.environ.get('INVISIBLEWATERMARK_PACKAGE', "git+https://github.com/patrickvonplaten/invisible-watermark.git@remove_onnxruntime_depedency") + install(invisiblewatermark_package, 'invisible-watermark') install('onnxruntime==1.15.1', 'onnxruntime', ignore=True) if args.profile: print_profile(pr, 'Packages') @@ -417,7 +429,7 @@ def install_repositories(): pr.enable() def d(name): return os.path.join(os.path.dirname(__file__), 'repositories', name) - log.info('Installing repositories') + log.info('Verifying repositories') os.makedirs(os.path.join(os.path.dirname(__file__), 'repositories'), exist_ok=True) stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") # stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf") @@ -529,9 +541,9 @@ def install_submodules(): if args.profile: pr = cProfile.Profile() pr.enable() - log.info('Installing submodules') + log.info('Verifying submodules') txt = git('submodule') - log.debug(f'Submodules list: {txt}') + # log.debug(f'Submodules list: {txt}') if 'no submodule mapping found' in txt: log.warning('Attempting repository recover') git('add .') @@ -543,15 +555,16 @@ def install_submodules(): txt = git('submodule') log.info('Continuing setup') git('submodule --quiet update --init --recursive') - if args.upgrade: - log.info('Updating submodules') - submodules = txt.splitlines() - for submodule in submodules: - try: - name = submodule.split()[1].strip() + submodules = txt.splitlines() + for submodule in submodules: + try: + name = submodule.split()[1].strip() + if args.upgrade: update(name) - except Exception: - log.error(f'Error updating submodule: {submodule}') + else: + branch(name) + except Exception: + log.error(f'Error updating submodule: {submodule}') if args.profile: print_profile(pr, 'Submodule') @@ -656,7 +669,7 @@ def check_version(offline=False, reset=True): # pylint: disable=unused-argument try: git('add .') git('stash') - update('.') + update('.', current_branch=True) # git('git stash pop') ver = git('log -1 --pretty=format:"%h %ad"') log.info(f'Upgraded to version: {ver}') @@ -678,7 +691,6 @@ def update_wiki(): log.info('Updating Wiki') try: update(os.path.join(os.path.dirname(__file__), "wiki")) - update(os.path.join(os.path.dirname(__file__), "wiki", "origin-wiki")) except Exception: log.error('Error updating wiki') diff --git a/javascript/black-orange.css b/javascript/black-orange.css index 6f07773f7..20e9f7ec6 100644 --- a/javascript/black-orange.css +++ b/javascript/black-orange.css @@ -1,43 +1,58 @@ /* generic html tags */ -:root { --font: "Source Sans Pro", 'ui-sans-serif', 'system-ui', "Roboto", sans-serif; } -html { font-size: 16px; } +:root { + --font: "Source Sans Pro", 'ui-sans-serif', 'system-ui', "Roboto", sans-serif; + --font-size: 16px; + --left-column: 490px; + --highlight-color: #ce6400; + --inactive-color: #4e1400; + --background-color: #000000; + --primary-50: #fff7ed; + --primary-100: #ffedd5; + --primary-200: #fed7aa; + --primary-300: #fdba74; + --primary-400: #fb923c; + --primary-500: #f97316; + --primary-600: #ea580c; + --primary-700: #c2410c; + --primary-800: #9a3412; + --primary-900: #7c2d12; + --primary-950: #6c2e12; +} +.light, .dark { + --radius-lg: 2px; + --radius-sm: 1px; + --spacing-md: 5px; +} + +html { font-size: var(--font-size); } body, button, input, select, textarea { font-family: var(--font);} button { font-size: 1.2rem; } -img { background-color: black; } -input[type=range] { height: 18px; appearance: none; margin-top: 0; min-width: 160px; background-color: black; width: 100%; background: transparent; } -input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: #50555C; border-radius: 2px; border: 0px solid #222222; } -input[type=range]::-moz-range-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: #50555C; border-radius: 2px; border: 0px solid #222222; } -input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: 2px; background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; } -input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: 2px; background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; } +img { background-color: var(--background-color); } +input[type=range] { height: 18px; appearance: none; margin-top: 0; min-width: 160px; background-color: var(--background-color); width: 100%; background: transparent; } +input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: #50555C; border-radius: var(--radius-lg); border: 0px solid #222222; } +input[type=range]::-moz-range-track { width: 100%; height: 18px; cursor: pointer; box-shadow: 2px 2px 3px #111111; background: #50555C; border-radius: var(--radius-lg); border: 0px solid #222222; } +input[type=range]::-webkit-slider-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: var(--radius-lg); background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; } +input[type=range]::-moz-range-thumb { box-shadow: 2px 2px 3px #111111; border: 0px solid #000000; height: 18px; width: 40px; border-radius: var(--radius-lg); background: var(--highlight-color); cursor: pointer; appearance: none; margin-top: 0px; } ::-webkit-scrollbar { width: 12px; } ::-webkit-scrollbar-track { background: #333333; } -::-webkit-scrollbar-thumb { background-color: var(--highlight-color); border-radius: 2px; border-width: 0; box-shadow: 2px 2px 3px #111111; } +::-webkit-scrollbar-thumb { background-color: var(--highlight-color); border-radius: var(--radius-lg); border-width: 0; box-shadow: 2px 2px 3px #111111; } div.form { border-width: 0; box-shadow: none; background: transparent; overflow: visible; gap: 0.5em; margin-bottom: 6px; } - -/* main gradio components by selector */ -div.gradio-container.dark > div.w-full.flex.flex-col.min-h-screen > div { background-color: black; } - -/* gradio shadowroot */ -.gradio-container { font-family: var(--font); --left-column: 490px; --highlight-color: #CE6400; --inactive-color: #4E1400; } +div.compact { gap: 0.8em; } /* gradio style classes */ +fieldset .gr-block.gr-box, label.block span { padding: 0; margin-top: -4px; } .border-2 { border-width: 0; } .border-b-2 { border-bottom-width: 2px; border-color: var(--highlight-color) !important; padding-bottom: 2px; margin-bottom: 8px; } -.dark .bg-white { color: lightyellow; border-radius: 0; background-color: var(--inactive-color); } -.dark .bg-gray-200, .dark .\!bg-gray-200 { background-color: transparent; } -.dark .dark\:bg-gray-900 { background-color: black; } -.dark .gr-box { border-radius: 0 !important; background-color: #111111 !important; box-shadow: 2px 2px 3px #111111; border-width: 0; padding: 4px; margin: 12px 0px 12px 0px } -.dark .gr-button { border-radius: 0; font-weight: normal; box-shadow: 2px 2px 3px #111111; font-size: 0.8rem; min-width: 32px; min-height: 32px; padding: 3px; margin: 3px; } -.dark .gr-check-radio { background-color: var(--inactive-color); border-width: 0; border-radius: 2px; box-shadow: 2px 2px 3px #111111; } -.dark .gr-check-radio:checked { background-color: var(--highlight-color); } -.dark .gr-compact { border-radius: 0; background-color: black; } -.dark .gr-form { border-radius: 0; border-width: 0; } -.dark .gr-input { background-color: #333333 !important; padding: 4px; margin: 4px; } -.dark .gr-input-label { color: lightyellow; border-width: 0; background: transparent; padding: 2px !important; } -.dark .gr-panel { border-radius: 0; background-color: black; } -.dark { background-color: black; } -.dark fieldset span.text-gray-500, .dark .gr-block.gr-box span.text-gray-500, .dark label.block span { padding: 0; margin-top: -4px; } -.dark fieldset span.text-gray-500, .dark .gr-block.gr-box span.text-gray-500, .dark label.block span { border-radius: 0;} +.bg-white { color: lightyellow; background-color: var(--inactive-color); } +.gr-box { border-radius: var(--radius-sm) !important; background-color: #111111 !important; box-shadow: 2px 2px 3px #111111; border-width: 0; padding: 4px; margin: 12px 0px 12px 0px } +.gr-button { font-weight: normal; box-shadow: 2px 2px 3px #111111; font-size: 0.8rem; min-width: 32px; min-height: 32px; padding: 3px; margin: 3px; } +.gr-check-radio { background-color: var(--inactive-color); border-width: 0; border-radius: var(--radius-lg); box-shadow: 2px 2px 3px #111111; } +.gr-check-radio:checked { background-color: var(--highlight-color); } +.gr-compact { background-color: var(--background-color); } +.gr-form { border-width: 0; } +.gr-input { background-color: #333333 !important; padding: 4px; margin: 4px; } +.gr-input-label { color: lightyellow; border-width: 0; background: transparent; padding: 2px !important; } +.gr-panel { background-color: var(--background-color); } .eta-bar { display: none !important } svg.feather.feather-image, .feather .feather-image { display: none } .gap-2 { padding-top: 8px; } @@ -48,11 +63,9 @@ svg.feather.feather-image, .feather .feather-image { display: none } .p-2 { padding: 0; } .px-4 { padding-lefT: 1rem; padding-right: 1rem; } .py-6 { padding-bottom: 0; } -.rounded-lg { border-radius: 0; } -.tabs { background-color: black; } -.gradio-button.tool { border-radius: 0; } -.block.token-counter span { background-color: #222 !important; box-shadow: 2px 2px 2px #111; border: none !important; border-radius: 0; font-size: 0.8rem; } -.tab-nav { zoom: 120%; margin-bottom: 10px; border-bottom: 2px solid #CE6400 !important; padding-bottom: 2px; } +.tabs { background-color: var(--background-color); } +.block.token-counter span { background-color: #222 !important; box-shadow: 2px 2px 2px #111; border: none !important; font-size: 0.8rem; } +.tab-nav { zoom: 120%; margin-bottom: 10px; border-bottom: 2px solid var(--highlight-color) !important; padding-bottom: 2px; } .label-wrap { margin: 16px 0px 8px 0px; } .gradio-slider input[type="number"] { width: 4.5em; font-size: 0.8rem; } #tab_extensions table td, #tab_extensions table th { border: none; padding: 0.5em; } @@ -60,26 +73,25 @@ svg.feather.feather-image, .feather .feather-image { display: none } #tab_extensions table thead { background-color: var(--neutral-700); } /* automatic style classes */ -.progressDiv { border-radius: 0 !important; position: fixed; top: 44px; right: 26px; max-width: 262px; height: 48px; z-index: 99; box-shadow: var(--button-shadow); } -.progressDiv .progress { border-radius: 0 !important; background: var(--highlight-color); line-height: 3rem; height: 48px; } +.progressDiv { border-radius: var(--radius-sm) !important; position: fixed; top: 44px; right: 26px; max-width: 262px; height: 48px; z-index: 99; box-shadow: var(--button-shadow); } +.progressDiv .progress { border-radius: var(--radius-lg) !important; background: var(--highlight-color); line-height: 3rem; height: 48px; } .gallery-item { box-shadow: none !important; } .performance { color: #888; } -.extra-networks { border-left: 2px solid #CE6400 !important; padding-left: 4px; } +.extra-networks { border-left: 2px solid var(--highlight-color) !important; padding-left: 4px; } .image-buttons { gap: 10px !important} /* gradio elements overrides */ -#div.gradio-container.dark { overflow-x: hidden; } +#div.gradio-container { overflow-x: hidden; } #img2img_label_copy_to_img2img { font-weight: normal; } -#txt2img_prompt, #txt2img_neg_prompt, #img2img_prompt, #img2img_neg_prompt { background-color: black; box-shadow: 4px 4px 4px 0px #333333 !important; } -#txt2img_prompt > label > textarea, #txt2img_neg_prompt > label > textarea, #img2img_prompt > label > textarea, #img2img_neg_prompt > label > textarea { font-size: 1.2rem; } +#txt2img_prompt, #txt2img_neg_prompt, #img2img_prompt, #img2img_neg_prompt { background-color: var(--background-color); box-shadow: 4px 4px 4px 0px #333333 !important; } +#txt2img_prompt > label > textarea, #txt2img_neg_prompt > label > textarea, #img2img_prompt > label > textarea, #img2img_neg_prompt > label > textarea { font-size: 1.1rem; } #img2img_settings { min-width: calc(2 * var(--left-column)); max-width: calc(2 * var(--left-column)); background-color: #111111; padding-top: 16px; } #interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; } #lightboxModal { background-color: rgba(20, 20, 20, 0.8) } #quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-left: -20px; margin-top: -2px; height: 2.4em; } #quicksettings > div, #quicksettings > fieldset { min-width: 24em; max-width: 26em; line-height: 2em; } -#refresh_sd_model_checkpoint { height: 48px; margin-left: -14px; background: #333333; box-shadow: none; } #open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; } -#save-animation { border-radius: 0 !important; margin-bottom: 16px; background-color: #111111; } +#save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: #111111; } #script_list { padding: 4px; margin-top: 20px; margin-bottom: 20px; } #settings > div.flex-wrap { width: 15em; } #tab_extensions table { background-color: #222222; } @@ -90,7 +102,7 @@ svg.feather.feather-image, .feather .feather-image { display: none } #extras_upscale { margin-top: 10px } #txt2img_progress_row > div { min-width: var(--left-column); max-width: var(--left-column); } -#txt2img_results, #img2img_results, #extras_results { background-color: black; padding: 0; } +#txt2img_results, #img2img_results, #extras_results { background-color: var(--background-color); padding: 0; } #txt2img_seed_row { padding: 0; margin-top: 8px; } #txt2img_settings { min-width: var(--left-column); max-width: var(--left-column); background-color: #111111; padding-top: 16px; } #txt2img_subseed_row { padding: 0; margin-top: 16px; } @@ -104,14 +116,14 @@ svg.feather.feather-image, .feather .feather-image { display: none } #steps-animation, #controlnet { border-width: 0; } /* based on gradio built-in dark theme */ -.dark { - --body-background-fill: black; +:root, .light, .dark { + --body-background-fill: var(--background-color); --body-text-color: var(--neutral-100); --color-accent-soft: var(--neutral-700); --background-fill-primary: #222222; --background-fill-secondary: none; - --border-color-accent: black; - --border-color-primary: black; + --border-color-accent: var(--background-color); + --border-color-primary: var(--background-color); --link-text-color-active: var(--secondary-500); --link-text-color: var(--secondary-500); --link-text-color-hover: var(--secondary-400); @@ -191,17 +203,6 @@ svg.feather.feather-image, .feather .feather-image { display: none } --button-secondary-border-color-hover: var(--button-secondary-border-color); --button-secondary-text-color: white; --button-secondary-text-color-hover: var(--button-secondary-text-color); - --primary-50: #fff7ed; - --primary-100: #ffedd5; - --primary-200: #fed7aa; - --primary-300: #fdba74; - --primary-400: #fb923c; - --primary-500: #f97316; - --primary-600: #ea580c; - --primary-700: #c2410c; - --primary-800: #9a3412; - --primary-900: #7c2d12; - --primary-950: #6c2e12; --secondary-50: #eff6ff; --secondary-100: #dbeafe; --secondary-200: #bfdbfe; @@ -227,15 +228,12 @@ svg.feather.feather-image, .feather .feather-image { display: none } --spacing-xxs: 1px; --spacing-xs: 2px; --spacing-sm: 4px; - --spacing-md: 6px; --spacing-lg: 8px; --spacing-xl: 10px; --spacing-xxl: 18px; --radius-xxs: 0; --radius-xs: 0; - --radius-sm: 0; --radius-md: 0; - --radius-lg: 0; --radius-xl: 0; --radius-xxl: 0; --text-xxs: 9px; @@ -282,7 +280,7 @@ svg.feather.feather-image, .feather .feather-image { display: none } --section-header-text-weight: 400; --checkbox-border-radius: var(--radius-sm); --checkbox-label-gap: 2px; - --checkbox-label-padding: var(--spacing-md) calc(2 * var(--spacing-md)); + --checkbox-label-padding: var(--spacing-md); --checkbox-label-shadow: var(--shadow-drop); --checkbox-label-text-size: var(--text-md); --checkbox-label-text-weight: 400; diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index f2e763362..ad0babd6a 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -9,6 +9,7 @@ function setupExtraNetworksForTab(tabname) { const refresh = gradioApp().getElementById(`${tabname}_extra_refresh`); const description = gradioApp().getElementById(`${tabname}_description`); const close = gradioApp().getElementById(`${tabname}_extra_close`); + const en = gradioApp().getElementById(`${tabname}_extra_networks`); search.classList.add('search'); description.classList.add('description'); tabs.appendChild(refresh); @@ -19,48 +20,46 @@ function setupExtraNetworksForTab(tabname) { searchTerm = search.value.toLowerCase(); gradioApp().querySelectorAll(`#${tabname}_extra_tabs div.card`).forEach((elem) => { text = `${elem.querySelector('.name').textContent.toLowerCase()} ${elem.querySelector('.search_term').textContent.toLowerCase()}`; + text = text.replace('models--', 'Diffusers') elem.style.display = text.indexOf(searchTerm) == -1 ? 'none' : ''; }); }); + intersectionObserver = new IntersectionObserver((entries) => { - // if (entries[0].intersectionRatio <= 0) onHidden(); - const en = gradioApp().getElementById(`${tabname}_extra_networks`); + if (!en) return + for (el of Array.from(gradioApp().querySelectorAll('.extra-networks-page'))) el.style.height = window.opts.extra_networks_height + 'vh'; if (entries[0].intersectionRatio > 0) { - for (el of Array.from(gradioApp().querySelectorAll('.extra-network-cards'))) { - const rect = el.getBoundingClientRect(); + if (window.opts.extra_networks_card_cover === 'cover') { + en.style.transition = ''; + en.style.zIndex = 9999; + en.style.position = 'absolute'; + en.style.right = 'unset'; + en.style.width = 'unset'; + en.style.height = 'unset'; + gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' + } else if (window.opts.extra_networks_card_cover === 'sidebar') { en.style.transition = 'width 0.2s ease'; - if (rect.top > 0) { - if (!en) return - if (window.opts.extra_networks_card_cover == 'cover') { - en.style.zIndex = 9999; - en.style.position = 'absolute'; - en.style.right = 'unset'; - en.style.width = 'unset'; - el.style.height = document.body.offsetHeight - el.getBoundingClientRect().top + 'px'; - gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' - } if (window.opts.extra_networks_card_cover == 'sidebar') { - en.style.zIndex = 0; - en.style.position = 'absolute'; - en.style.right = '0'; - en.style.width = window.opts.extra_networks_sidebar_width + 'vw'; - el.style.height = gradioApp().getElementById(`${tabname}_settings`).offsetHeight - 90 + 'px'; - gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 100 - 2 - window.opts.extra_networks_sidebar_width + 'vw'; - } else { - en.style.zIndex = 0; - en.style.position = 'relative'; - en.style.right = 'unset'; - en.style.width = 'unset'; - el.style.height = window.innerHeight - el.getBoundingClientRect().top + 'px'; - gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' - } - } + en.style.zIndex = 0; + en.style.position = 'absolute'; + en.style.right = '0'; + en.style.width = window.opts.extra_networks_sidebar_width + 'vw'; + en.style.height = '-webkit-fill-available' + gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 100 - 2 - window.opts.extra_networks_sidebar_width + 'vw'; + } else { + en.style.transition = ''; + en.style.zIndex = 0; + en.style.position = 'relative'; + en.style.right = 'unset'; + en.style.width = 'unset'; + en.style.height = 'unset'; + gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' } } else { en.style.width = 0; gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' } }); - intersectionObserver.observe(search); // monitor visibility of + intersectionObserver.observe(en); // monitor visibility of } function setupExtraNetworks() { diff --git a/javascript/imageViewer.js b/javascript/imageViewer.js index c4cac2f90..b046b0503 100644 --- a/javascript/imageViewer.js +++ b/javascript/imageViewer.js @@ -149,9 +149,7 @@ onAfterUiUpdate(() => { }); document.addEventListener('DOMContentLoaded', () => { - // const modalFragment = document.createDocumentFragment(); const modal = document.createElement('div'); - modal.onclick = closeModal; modal.id = 'lightboxModal'; modal.tabIndex = 0; modal.addEventListener('keydown', modalKeyHandler, true); @@ -182,22 +180,19 @@ document.addEventListener('DOMContentLoaded', () => { modalSave.title = 'Save Image(s)'; modalControls.appendChild(modalSave); - /* - const modalClose = document.createElement('span'); - modalClose.className = 'modalClose cursor'; - modalClose.innerHTML = '×'; - modalClose.onclick = closeModal; - modalClose.title = 'Close image viewer'; - modalControls.appendChild(modalClose); - */ - const modalImage = document.createElement('img'); modalImage.id = 'modalImage'; - modalImage.onclick = closeModal; modalImage.tabIndex = 0; modalImage.addEventListener('keydown', modalKeyHandler, true); modal.appendChild(modalImage); + modalImage.onload = () => panzoom(modalImage, { zoomSpeed: 0.025, minZoom: 0.25, maxZoom: 4.0 }); + let drag = false; + modal.addEventListener('mousedown', () => drag = false); + modal.addEventListener('mousemove', () => drag = true); + modal.addEventListener('scroll', () => drag = true); + modal.addEventListener('mouseup', () => { if (!drag) closeModal(); }); + const modalPrev = document.createElement('a'); modalPrev.className = 'modalPrev'; modalPrev.innerHTML = '❮'; diff --git a/javascript/panZoom.js b/javascript/panZoom.js new file mode 100644 index 000000000..fec42c597 --- /dev/null +++ b/javascript/panZoom.js @@ -0,0 +1,1768 @@ +(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.panzoom = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i 0) { + transform.x += diff; + adjusted = true; + } + // check the other side: + diff = boundingBox.right - clientRect.left; + if (diff < 0) { + transform.x += diff; + adjusted = true; + } + + // y axis: + diff = boundingBox.top - clientRect.bottom; + if (diff > 0) { + // we adjust transform, so that it matches exactly our bounding box: + // transform.y = boundingBox.top - (boundingBox.height + boundingBox.y) * transform.scale => + // transform.y = boundingBox.top - (clientRect.bottom - transform.y) => + // transform.y = diff + transform.y => + transform.y += diff; + adjusted = true; + } + + diff = boundingBox.bottom - clientRect.top; + if (diff < 0) { + transform.y += diff; + adjusted = true; + } + return adjusted; + } + + /** + * Returns bounding box that should be used to restrict scene movement. + */ + function getBoundingBox() { + if (!bounds) return; // client does not want to restrict movement + + if (typeof bounds === 'boolean') { + // for boolean type we use parent container bounds + var ownerRect = owner.getBoundingClientRect(); + var sceneWidth = ownerRect.width; + var sceneHeight = ownerRect.height; + + return { + left: sceneWidth * boundsPadding, + top: sceneHeight * boundsPadding, + right: sceneWidth * (1 - boundsPadding), + bottom: sceneHeight * (1 - boundsPadding) + }; + } + + return bounds; + } + + function getClientRect() { + var bbox = panController.getBBox(); + var leftTop = client(bbox.left, bbox.top); + + return { + left: leftTop.x, + top: leftTop.y, + right: bbox.width * transform.scale + leftTop.x, + bottom: bbox.height * transform.scale + leftTop.y + }; + } + + function client(x, y) { + return { + x: x * transform.scale + transform.x, + y: y * transform.scale + transform.y + }; + } + + function makeDirty() { + isDirty = true; + frameAnimation = window.requestAnimationFrame(frame); + } + + function zoomByRatio(clientX, clientY, ratio) { + if (isNaN(clientX) || isNaN(clientY) || isNaN(ratio)) { + throw new Error('zoom requires valid numbers'); + } + + var newScale = transform.scale * ratio; + + if (newScale < minZoom) { + if (transform.scale === minZoom) return; + + ratio = minZoom / transform.scale; + } + if (newScale > maxZoom) { + if (transform.scale === maxZoom) return; + + ratio = maxZoom / transform.scale; + } + + var size = transformToScreen(clientX, clientY); + + transform.x = size.x - ratio * (size.x - transform.x); + transform.y = size.y - ratio * (size.y - transform.y); + + // TODO: https://github.com/anvaka/panzoom/issues/112 + if (bounds && boundsPadding === 1 && minZoom === 1) { + transform.scale *= ratio; + keepTransformInsideBounds(); + } else { + var transformAdjusted = keepTransformInsideBounds(); + if (!transformAdjusted) transform.scale *= ratio; + } + + triggerEvent('zoom'); + + makeDirty(); + } + + function zoomAbs(clientX, clientY, zoomLevel) { + var ratio = zoomLevel / transform.scale; + zoomByRatio(clientX, clientY, ratio); + } + + function centerOn(ui) { + var parent = ui.ownerSVGElement; + if (!parent) + throw new Error('ui element is required to be within the scene'); + + // TODO: should i use controller's screen CTM? + var clientRect = ui.getBoundingClientRect(); + var cx = clientRect.left + clientRect.width / 2; + var cy = clientRect.top + clientRect.height / 2; + + var container = parent.getBoundingClientRect(); + var dx = container.width / 2 - cx; + var dy = container.height / 2 - cy; + + internalMoveBy(dx, dy, true); + } + + function smoothMoveTo(x, y){ + internalMoveBy(x - transform.x, y - transform.y, true) + } + + function internalMoveBy(dx, dy, smooth) { + if (!smooth) { + return moveBy(dx, dy); + } + + if (moveByAnimation) moveByAnimation.cancel(); + + var from = { x: 0, y: 0 }; + var to = { x: dx, y: dy }; + var lastX = 0; + var lastY = 0; + + moveByAnimation = animate(from, to, { + step: function (v) { + moveBy(v.x - lastX, v.y - lastY); + + lastX = v.x; + lastY = v.y; + } + }); + } + + function scroll(x, y) { + cancelZoomAnimation(); + moveTo(x, y); + } + + function dispose() { + releaseEvents(); + } + + function listenForEvents() { + owner.addEventListener('mousedown', onMouseDown, { passive: false }); + owner.addEventListener('dblclick', onDoubleClick, { passive: false }); + owner.addEventListener('touchstart', onTouch, { passive: false }); + owner.addEventListener('keydown', onKeyDown, { passive: false }); + + // Need to listen on the owner container, so that we are not limited + // by the size of the scrollable domElement + wheel.addWheelListener(owner, onMouseWheel, { passive: false }); + + makeDirty(); + } + + function releaseEvents() { + wheel.removeWheelListener(owner, onMouseWheel); + owner.removeEventListener('mousedown', onMouseDown); + owner.removeEventListener('keydown', onKeyDown); + owner.removeEventListener('dblclick', onDoubleClick); + owner.removeEventListener('touchstart', onTouch); + + if (frameAnimation) { + window.cancelAnimationFrame(frameAnimation); + frameAnimation = 0; + } + + smoothScroll.cancel(); + + releaseDocumentMouse(); + releaseTouches(); + textSelection.release(); + + triggerPanEnd(); + } + + function frame() { + if (isDirty) applyTransform(); + } + + function applyTransform() { + isDirty = false; + + // TODO: Should I allow to cancel this? + panController.applyTransform(transform); + + triggerEvent('transform'); + frameAnimation = 0; + } + + function onKeyDown(e) { + var x = 0, + y = 0, + z = 0; + if (e.keyCode === 38) { + y = 1; // up + } else if (e.keyCode === 40) { + y = -1; // down + } else if (e.keyCode === 37) { + x = 1; // left + } else if (e.keyCode === 39) { + x = -1; // right + } else if (e.keyCode === 189 || e.keyCode === 109) { + // DASH or SUBTRACT + z = 1; // `-` - zoom out + } else if (e.keyCode === 187 || e.keyCode === 107) { + // EQUAL SIGN or ADD + z = -1; // `=` - zoom in (equal sign on US layout is under `+`) + } + + if (filterKey(e, x, y, z)) { + // They don't want us to handle the key: https://github.com/anvaka/panzoom/issues/45 + return; + } + + if (x || y) { + e.preventDefault(); + e.stopPropagation(); + + var clientRect = owner.getBoundingClientRect(); + // movement speed should be the same in both X and Y direction: + var offset = Math.min(clientRect.width, clientRect.height); + var moveSpeedRatio = 0.05; + var dx = offset * moveSpeedRatio * x; + var dy = offset * moveSpeedRatio * y; + + // TODO: currently we do not animate this. It could be better to have animation + internalMoveBy(dx, dy); + } + + if (z) { + var scaleMultiplier = getScaleMultiplier(z * 100); + var offset = transformOrigin ? getTransformOriginOffset() : midPoint(); + publicZoomTo(offset.x, offset.y, scaleMultiplier); + } + } + + function midPoint() { + var ownerRect = owner.getBoundingClientRect(); + return { + x: ownerRect.width / 2, + y: ownerRect.height / 2 + }; + } + + function onTouch(e) { + // let the override the touch behavior + beforeTouch(e); + + if (e.touches.length === 1) { + return handleSingleFingerTouch(e, e.touches[0]); + } else if (e.touches.length === 2) { + // handleTouchMove() will care about pinch zoom. + pinchZoomLength = getPinchZoomLength(e.touches[0], e.touches[1]); + multiTouch = true; + startTouchListenerIfNeeded(); + } + } + + function beforeTouch(e) { + // TODO: Need to unify this filtering names. E.g. use `beforeTouch` + if (options.onTouch && !options.onTouch(e)) { + // if they return `false` from onTouch, we don't want to stop + // events propagation. Fixes https://github.com/anvaka/panzoom/issues/12 + return; + } + + e.stopPropagation(); + e.preventDefault(); + } + + function beforeDoubleClick(e) { + // TODO: Need to unify this filtering names. E.g. use `beforeDoubleClick`` + if (options.onDoubleClick && !options.onDoubleClick(e)) { + // if they return `false` from onTouch, we don't want to stop + // events propagation. Fixes https://github.com/anvaka/panzoom/issues/46 + return; + } + + e.preventDefault(); + e.stopPropagation(); + } + + function handleSingleFingerTouch(e) { + var touch = e.touches[0]; + var offset = getOffsetXY(touch); + lastSingleFingerOffset = offset; + var point = transformToScreen(offset.x, offset.y); + mouseX = point.x; + mouseY = point.y; + + smoothScroll.cancel(); + startTouchListenerIfNeeded(); + } + + function startTouchListenerIfNeeded() { + if (touchInProgress) { + // no need to do anything, as we already listen to events; + return; + } + + touchInProgress = true; + document.addEventListener('touchmove', handleTouchMove); + document.addEventListener('touchend', handleTouchEnd); + document.addEventListener('touchcancel', handleTouchEnd); + } + + function handleTouchMove(e) { + if (e.touches.length === 1) { + e.stopPropagation(); + var touch = e.touches[0]; + + var offset = getOffsetXY(touch); + var point = transformToScreen(offset.x, offset.y); + + var dx = point.x - mouseX; + var dy = point.y - mouseY; + + if (dx !== 0 && dy !== 0) { + triggerPanStart(); + } + mouseX = point.x; + mouseY = point.y; + internalMoveBy(dx, dy); + } else if (e.touches.length === 2) { + // it's a zoom, let's find direction + multiTouch = true; + var t1 = e.touches[0]; + var t2 = e.touches[1]; + var currentPinchLength = getPinchZoomLength(t1, t2); + + // since the zoom speed is always based on distance from 1, we need to apply + // pinch speed only on that distance from 1: + var scaleMultiplier = + 1 + (currentPinchLength / pinchZoomLength - 1) * pinchSpeed; + + var firstTouchPoint = getOffsetXY(t1); + var secondTouchPoint = getOffsetXY(t2); + mouseX = (firstTouchPoint.x + secondTouchPoint.x) / 2; + mouseY = (firstTouchPoint.y + secondTouchPoint.y) / 2; + if (transformOrigin) { + var offset = getTransformOriginOffset(); + mouseX = offset.x; + mouseY = offset.y; + } + + publicZoomTo(mouseX, mouseY, scaleMultiplier); + + pinchZoomLength = currentPinchLength; + e.stopPropagation(); + e.preventDefault(); + } + } + + function handleTouchEnd(e) { + if (e.touches.length > 0) { + var offset = getOffsetXY(e.touches[0]); + var point = transformToScreen(offset.x, offset.y); + mouseX = point.x; + mouseY = point.y; + } else { + var now = new Date(); + if (now - lastTouchEndTime < doubleTapSpeedInMS) { + if (transformOrigin) { + var offset = getTransformOriginOffset(); + smoothZoom(offset.x, offset.y, zoomDoubleClickSpeed); + } else { + // We want untransformed x/y here. + smoothZoom(lastSingleFingerOffset.x, lastSingleFingerOffset.y, zoomDoubleClickSpeed); + } + } + + lastTouchEndTime = now; + + triggerPanEnd(); + releaseTouches(); + } + } + + function getPinchZoomLength(finger1, finger2) { + var dx = finger1.clientX - finger2.clientX; + var dy = finger1.clientY - finger2.clientY; + return Math.sqrt(dx * dx + dy * dy); + } + + function onDoubleClick(e) { + beforeDoubleClick(e); + var offset = getOffsetXY(e); + if (transformOrigin) { + // TODO: looks like this is duplicated in the file. + // Need to refactor + offset = getTransformOriginOffset(); + } + smoothZoom(offset.x, offset.y, zoomDoubleClickSpeed); + } + + function onMouseDown(e) { + // if client does not want to handle this event - just ignore the call + if (beforeMouseDown(e)) return; + + if (touchInProgress) { + // modern browsers will fire mousedown for touch events too + // we do not want this: touch is handled separately. + e.stopPropagation(); + return false; + } + // for IE, left click == 1 + // for Firefox, left click == 0 + var isLeftButton = + (e.button === 1 && window.event !== null) || e.button === 0; + if (!isLeftButton) return; + + smoothScroll.cancel(); + + var offset = getOffsetXY(e); + var point = transformToScreen(offset.x, offset.y); + mouseX = point.x; + mouseY = point.y; + + // We need to listen on document itself, since mouse can go outside of the + // window, and we will loose it + document.addEventListener('mousemove', onMouseMove); + document.addEventListener('mouseup', onMouseUp); + textSelection.capture(e.target || e.srcElement); + + return false; + } + + function onMouseMove(e) { + // no need to worry about mouse events when touch is happening + if (touchInProgress) return; + + triggerPanStart(); + + var offset = getOffsetXY(e); + var point = transformToScreen(offset.x, offset.y); + var dx = point.x - mouseX; + var dy = point.y - mouseY; + + mouseX = point.x; + mouseY = point.y; + + internalMoveBy(dx, dy); + } + + function onMouseUp() { + textSelection.release(); + triggerPanEnd(); + releaseDocumentMouse(); + } + + function releaseDocumentMouse() { + document.removeEventListener('mousemove', onMouseMove); + document.removeEventListener('mouseup', onMouseUp); + panstartFired = false; + } + + function releaseTouches() { + document.removeEventListener('touchmove', handleTouchMove); + document.removeEventListener('touchend', handleTouchEnd); + document.removeEventListener('touchcancel', handleTouchEnd); + panstartFired = false; + multiTouch = false; + touchInProgress = false; + } + + function onMouseWheel(e) { + // if client does not want to handle this event - just ignore the call + if (beforeWheel(e)) return; + + smoothScroll.cancel(); + + var delta = e.deltaY; + if (e.deltaMode > 0) delta *= 100; + + var scaleMultiplier = getScaleMultiplier(delta); + + if (scaleMultiplier !== 1) { + var offset = transformOrigin + ? getTransformOriginOffset() + : getOffsetXY(e); + publicZoomTo(offset.x, offset.y, scaleMultiplier); + e.preventDefault(); + } + } + + function getOffsetXY(e) { + var offsetX, offsetY; + // I tried using e.offsetX, but that gives wrong results for svg, when user clicks on a path. + var ownerRect = owner.getBoundingClientRect(); + offsetX = e.clientX - ownerRect.left; + offsetY = e.clientY - ownerRect.top; + + return { x: offsetX, y: offsetY }; + } + + function smoothZoom(clientX, clientY, scaleMultiplier) { + var fromValue = transform.scale; + var from = { scale: fromValue }; + var to = { scale: scaleMultiplier * fromValue }; + + smoothScroll.cancel(); + cancelZoomAnimation(); + + zoomToAnimation = animate(from, to, { + step: function (v) { + zoomAbs(clientX, clientY, v.scale); + }, + done: triggerZoomEnd + }); + } + + function smoothZoomAbs(clientX, clientY, toScaleValue) { + var fromValue = transform.scale; + var from = { scale: fromValue }; + var to = { scale: toScaleValue }; + + smoothScroll.cancel(); + cancelZoomAnimation(); + + zoomToAnimation = animate(from, to, { + step: function (v) { + zoomAbs(clientX, clientY, v.scale); + } + }); + } + + function getTransformOriginOffset() { + var ownerRect = owner.getBoundingClientRect(); + return { + x: ownerRect.width * transformOrigin.x, + y: ownerRect.height * transformOrigin.y + }; + } + + function publicZoomTo(clientX, clientY, scaleMultiplier) { + smoothScroll.cancel(); + cancelZoomAnimation(); + return zoomByRatio(clientX, clientY, scaleMultiplier); + } + + function cancelZoomAnimation() { + if (zoomToAnimation) { + zoomToAnimation.cancel(); + zoomToAnimation = null; + } + } + + function getScaleMultiplier(delta) { + var sign = Math.sign(delta); + var deltaAdjustedSpeed = Math.min(0.25, Math.abs(speed * delta / 128)); + return 1 - sign * deltaAdjustedSpeed; + } + + function triggerPanStart() { + if (!panstartFired) { + triggerEvent('panstart'); + panstartFired = true; + smoothScroll.start(); + } + } + + function triggerPanEnd() { + if (panstartFired) { + // we should never run smooth scrolling if it was multiTouch (pinch zoom animation): + if (!multiTouch) smoothScroll.stop(); + triggerEvent('panend'); + } + } + + function triggerZoomEnd() { + triggerEvent('zoomend'); + } + + function triggerEvent(name) { + api.fire(name, api); + } +} + +function parseTransformOrigin(options) { + if (!options) return; + if (typeof options === 'object') { + if (!isNumber(options.x) || !isNumber(options.y)) + failTransformOrigin(options); + return options; + } + + failTransformOrigin(); +} + +function failTransformOrigin(options) { + console.error(options); + throw new Error( + [ + 'Cannot parse transform origin.', + 'Some good examples:', + ' "center center" can be achieved with {x: 0.5, y: 0.5}', + ' "top center" can be achieved with {x: 0.5, y: 0}', + ' "bottom right" can be achieved with {x: 1, y: 1}' + ].join('\n') + ); +} + +function noop() { } + +function validateBounds(bounds) { + var boundsType = typeof bounds; + if (boundsType === 'undefined' || boundsType === 'boolean') return; // this is okay + // otherwise need to be more thorough: + var validBounds = + isNumber(bounds.left) && + isNumber(bounds.top) && + isNumber(bounds.bottom) && + isNumber(bounds.right); + + if (!validBounds) + throw new Error( + 'Bounds object is not valid. It can be: ' + + 'undefined, boolean (true|false) or an object {left, top, right, bottom}' + ); +} + +function isNumber(x) { + return Number.isFinite(x); +} + +// IE 11 does not support isNaN: +function isNaN(value) { + if (Number.isNaN) { + return Number.isNaN(value); + } + + return value !== value; +} + +function rigidScroll() { + return { + start: noop, + stop: noop, + cancel: noop + }; +} + +function autoRun() { + if (typeof document === 'undefined') return; + + var scripts = document.getElementsByTagName('script'); + if (!scripts) return; + var panzoomScript; + + for (var i = 0; i < scripts.length; ++i) { + var x = scripts[i]; + if (x.src && x.src.match(/\bpanzoom(\.min)?\.js/)) { + panzoomScript = x; + break; + } + } + + if (!panzoomScript) return; + + var query = panzoomScript.getAttribute('query'); + if (!query) return; + + var globalName = panzoomScript.getAttribute('name') || 'pz'; + var started = Date.now(); + + tryAttach(); + + function tryAttach() { + var el = document.querySelector(query); + if (!el) { + var now = Date.now(); + var elapsed = now - started; + if (elapsed < 2000) { + // Let's wait a bit + setTimeout(tryAttach, 100); + return; + } + // If we don't attach within 2 seconds to the target element, consider it a failure + console.error('Cannot find the panzoom element', globalName); + return; + } + var options = collectOptions(panzoomScript); + console.log(options); + window[globalName] = createPanZoom(el, options); + } + + function collectOptions(script) { + var attrs = script.attributes; + var options = {}; + for (var i = 0; i < attrs.length; ++i) { + var attr = attrs[i]; + var nameValue = getPanzoomAttributeNameValue(attr); + if (nameValue) { + options[nameValue.name] = nameValue.value; + } + } + + return options; + } + + function getPanzoomAttributeNameValue(attr) { + if (!attr.name) return; + var isPanZoomAttribute = + attr.name[0] === 'p' && attr.name[1] === 'z' && attr.name[2] === '-'; + + if (!isPanZoomAttribute) return; + + var name = attr.name.substr(3); + var value = JSON.parse(attr.value); + return { name: name, value: value }; + } +} + +autoRun(); + +},{"./lib/createTextSelectionInterceptor.js":2,"./lib/domController.js":3,"./lib/kinetic.js":4,"./lib/svgController.js":5,"./lib/transform.js":6,"amator":7,"ngraph.events":9,"wheel":10}],2:[function(require,module,exports){ +/** + * Disallows selecting text. + */ +module.exports = createTextSelectionInterceptor; + +function createTextSelectionInterceptor(useFake) { + if (useFake) { + return { + capture: noop, + release: noop + }; + } + + var dragObject; + var prevSelectStart; + var prevDragStart; + var wasCaptured = false; + + return { + capture: capture, + release: release + }; + + function capture(domObject) { + wasCaptured = true; + prevSelectStart = window.document.onselectstart; + prevDragStart = window.document.ondragstart; + + window.document.onselectstart = disabled; + + dragObject = domObject; + dragObject.ondragstart = disabled; + } + + function release() { + if (!wasCaptured) return; + + wasCaptured = false; + window.document.onselectstart = prevSelectStart; + if (dragObject) dragObject.ondragstart = prevDragStart; + } +} + +function disabled(e) { + e.stopPropagation(); + return false; +} + +function noop() {} + +},{}],3:[function(require,module,exports){ +module.exports = makeDomController + +module.exports.canAttach = isDomElement; + +function makeDomController(domElement, options) { + var elementValid = isDomElement(domElement); + if (!elementValid) { + throw new Error('panzoom requires DOM element to be attached to the DOM tree') + } + + var owner = domElement.parentElement; + domElement.scrollTop = 0; + + if (!options.disableKeyboardInteraction) { + owner.setAttribute('tabindex', 0); + } + + var api = { + getBBox: getBBox, + getOwner: getOwner, + applyTransform: applyTransform, + } + + return api + + function getOwner() { + return owner + } + + function getBBox() { + // TODO: We should probably cache this? + return { + left: 0, + top: 0, + width: domElement.clientWidth, + height: domElement.clientHeight + } + } + + function applyTransform(transform) { + // TODO: Should we cache this? + domElement.style.transformOrigin = '0 0 0'; + domElement.style.transform = 'matrix(' + + transform.scale + ', 0, 0, ' + + transform.scale + ', ' + + transform.x + ', ' + transform.y + ')' + } +} + +function isDomElement(element) { + return element && element.parentElement && element.style; +} + +},{}],4:[function(require,module,exports){ +/** + * Allows smooth kinetic scrolling of the surface + */ +module.exports = kinetic; + +function kinetic(getPoint, scroll, settings) { + if (typeof settings !== 'object') { + // setting could come as boolean, we should ignore it, and use an object. + settings = {}; + } + + var minVelocity = typeof settings.minVelocity === 'number' ? settings.minVelocity : 5; + var amplitude = typeof settings.amplitude === 'number' ? settings.amplitude : 0.25; + var cancelAnimationFrame = typeof settings.cancelAnimationFrame === 'function' ? settings.cancelAnimationFrame : getCancelAnimationFrame(); + var requestAnimationFrame = typeof settings.requestAnimationFrame === 'function' ? settings.requestAnimationFrame : getRequestAnimationFrame(); + + var lastPoint; + var timestamp; + var timeConstant = 342; + + var ticker; + var vx, targetX, ax; + var vy, targetY, ay; + + var raf; + + return { + start: start, + stop: stop, + cancel: dispose + }; + + function dispose() { + cancelAnimationFrame(ticker); + cancelAnimationFrame(raf); + } + + function start() { + lastPoint = getPoint(); + + ax = ay = vx = vy = 0; + timestamp = new Date(); + + cancelAnimationFrame(ticker); + cancelAnimationFrame(raf); + + // we start polling the point position to accumulate velocity + // Once we stop(), we will use accumulated velocity to keep scrolling + // an object. + ticker = requestAnimationFrame(track); + } + + function track() { + var now = Date.now(); + var elapsed = now - timestamp; + timestamp = now; + + var currentPoint = getPoint(); + + var dx = currentPoint.x - lastPoint.x; + var dy = currentPoint.y - lastPoint.y; + + lastPoint = currentPoint; + + var dt = 1000 / (1 + elapsed); + + // moving average + vx = 0.8 * dx * dt + 0.2 * vx; + vy = 0.8 * dy * dt + 0.2 * vy; + + ticker = requestAnimationFrame(track); + } + + function stop() { + cancelAnimationFrame(ticker); + cancelAnimationFrame(raf); + + var currentPoint = getPoint(); + + targetX = currentPoint.x; + targetY = currentPoint.y; + timestamp = Date.now(); + + if (vx < -minVelocity || vx > minVelocity) { + ax = amplitude * vx; + targetX += ax; + } + + if (vy < -minVelocity || vy > minVelocity) { + ay = amplitude * vy; + targetY += ay; + } + + raf = requestAnimationFrame(autoScroll); + } + + function autoScroll() { + var elapsed = Date.now() - timestamp; + + var moving = false; + var dx = 0; + var dy = 0; + + if (ax) { + dx = -ax * Math.exp(-elapsed / timeConstant); + + if (dx > 0.5 || dx < -0.5) moving = true; + else dx = ax = 0; + } + + if (ay) { + dy = -ay * Math.exp(-elapsed / timeConstant); + + if (dy > 0.5 || dy < -0.5) moving = true; + else dy = ay = 0; + } + + if (moving) { + scroll(targetX + dx, targetY + dy); + raf = requestAnimationFrame(autoScroll); + } + } +} + +function getCancelAnimationFrame() { + if (typeof cancelAnimationFrame === 'function') return cancelAnimationFrame; + return clearTimeout; +} + +function getRequestAnimationFrame() { + if (typeof requestAnimationFrame === 'function') return requestAnimationFrame; + + return function (handler) { + return setTimeout(handler, 16); + } +} +},{}],5:[function(require,module,exports){ +module.exports = makeSvgController +module.exports.canAttach = isSVGElement; + +function makeSvgController(svgElement, options) { + if (!isSVGElement(svgElement)) { + throw new Error('svg element is required for svg.panzoom to work') + } + + var owner = svgElement.ownerSVGElement + if (!owner) { + throw new Error( + 'Do not apply panzoom to the root element. ' + + 'Use its child instead (e.g. ). ' + + 'As of March 2016 only FireFox supported transform on the root element') + } + + if (!options.disableKeyboardInteraction) { + owner.setAttribute('tabindex', 0); + } + + var api = { + getBBox: getBBox, + getScreenCTM: getScreenCTM, + getOwner: getOwner, + applyTransform: applyTransform, + initTransform: initTransform + } + + return api + + function getOwner() { + return owner + } + + function getBBox() { + var bbox = svgElement.getBBox() + return { + left: bbox.x, + top: bbox.y, + width: bbox.width, + height: bbox.height, + } + } + + function getScreenCTM() { + var ctm = owner.getCTM(); + if (!ctm) { + // This is likely firefox: https://bugzilla.mozilla.org/show_bug.cgi?id=873106 + // The code below is not entirely correct, but still better than nothing + return owner.getScreenCTM(); + } + return ctm; + } + + function initTransform(transform) { + var screenCTM = svgElement.getCTM() + + // The above line returns null on Firefox + if (screenCTM === null) { + screenCTM = document.createElementNS("http://www.w3.org/2000/svg", "svg").createSVGMatrix() + } + + transform.x = screenCTM.e; + transform.y = screenCTM.f; + transform.scale = screenCTM.a; + owner.removeAttributeNS(null, 'viewBox'); + } + + function applyTransform(transform) { + svgElement.setAttribute('transform', 'matrix(' + + transform.scale + ' 0 0 ' + + transform.scale + ' ' + + transform.x + ' ' + transform.y + ')') + } +} + +function isSVGElement(element) { + return element && element.ownerSVGElement && element.getCTM; +} +},{}],6:[function(require,module,exports){ +module.exports = Transform; + +function Transform() { + this.x = 0; + this.y = 0; + this.scale = 1; +} + +},{}],7:[function(require,module,exports){ +var BezierEasing = require('bezier-easing') + +// Predefined set of animations. Similar to CSS easing functions +var animations = { + ease: BezierEasing(0.25, 0.1, 0.25, 1), + easeIn: BezierEasing(0.42, 0, 1, 1), + easeOut: BezierEasing(0, 0, 0.58, 1), + easeInOut: BezierEasing(0.42, 0, 0.58, 1), + linear: BezierEasing(0, 0, 1, 1) +} + + +module.exports = animate; +module.exports.makeAggregateRaf = makeAggregateRaf; +module.exports.sharedScheduler = makeAggregateRaf(); + + +function animate(source, target, options) { + var start = Object.create(null) + var diff = Object.create(null) + options = options || {} + // We let clients specify their own easing function + var easing = (typeof options.easing === 'function') ? options.easing : animations[options.easing] + + // if nothing is specified, default to ease (similar to CSS animations) + if (!easing) { + if (options.easing) { + console.warn('Unknown easing function in amator: ' + options.easing); + } + easing = animations.ease + } + + var step = typeof options.step === 'function' ? options.step : noop + var done = typeof options.done === 'function' ? options.done : noop + + var scheduler = getScheduler(options.scheduler) + + var keys = Object.keys(target) + keys.forEach(function(key) { + start[key] = source[key] + diff[key] = target[key] - source[key] + }) + + var durationInMs = typeof options.duration === 'number' ? options.duration : 400 + var durationInFrames = Math.max(1, durationInMs * 0.06) // 0.06 because 60 frames pers 1,000 ms + var previousAnimationId + var frame = 0 + + previousAnimationId = scheduler.next(loop) + + return { + cancel: cancel + } + + function cancel() { + scheduler.cancel(previousAnimationId) + previousAnimationId = 0 + } + + function loop() { + var t = easing(frame/durationInFrames) + frame += 1 + setValues(t) + if (frame <= durationInFrames) { + previousAnimationId = scheduler.next(loop) + step(source) + } else { + previousAnimationId = 0 + setTimeout(function() { done(source) }, 0) + } + } + + function setValues(t) { + keys.forEach(function(key) { + source[key] = diff[key] * t + start[key] + }) + } +} + +function noop() { } + +function getScheduler(scheduler) { + if (!scheduler) { + var canRaf = typeof window !== 'undefined' && window.requestAnimationFrame + return canRaf ? rafScheduler() : timeoutScheduler() + } + if (typeof scheduler.next !== 'function') throw new Error('Scheduler is supposed to have next(cb) function') + if (typeof scheduler.cancel !== 'function') throw new Error('Scheduler is supposed to have cancel(handle) function') + + return scheduler +} + +function rafScheduler() { + return { + next: window.requestAnimationFrame.bind(window), + cancel: window.cancelAnimationFrame.bind(window) + } +} + +function timeoutScheduler() { + return { + next: function(cb) { + return setTimeout(cb, 1000/60) + }, + cancel: function (id) { + return clearTimeout(id) + } + } +} + +function makeAggregateRaf() { + var frontBuffer = new Set(); + var backBuffer = new Set(); + var frameToken = 0; + + return { + next: next, + cancel: next, + clearAll: clearAll + } + + function clearAll() { + frontBuffer.clear(); + backBuffer.clear(); + cancelAnimationFrame(frameToken); + frameToken = 0; + } + + function next(callback) { + backBuffer.add(callback); + renderNextFrame(); + } + + function renderNextFrame() { + if (!frameToken) frameToken = requestAnimationFrame(renderFrame); + } + + function renderFrame() { + frameToken = 0; + + var t = backBuffer; + backBuffer = frontBuffer; + frontBuffer = t; + + frontBuffer.forEach(function(callback) { + callback(); + }); + frontBuffer.clear(); + } + + function cancel(callback) { + backBuffer.delete(callback); + } +} + +},{"bezier-easing":8}],8:[function(require,module,exports){ +/** + * https://github.com/gre/bezier-easing + * BezierEasing - use bezier curve for transition easing function + * by Gaëtan Renaudeau 2014 - 2015 – MIT License + */ + +// These values are established by empiricism with tests (tradeoff: performance VS precision) +var NEWTON_ITERATIONS = 4; +var NEWTON_MIN_SLOPE = 0.001; +var SUBDIVISION_PRECISION = 0.0000001; +var SUBDIVISION_MAX_ITERATIONS = 10; + +var kSplineTableSize = 11; +var kSampleStepSize = 1.0 / (kSplineTableSize - 1.0); + +var float32ArraySupported = typeof Float32Array === 'function'; + +function A (aA1, aA2) { return 1.0 - 3.0 * aA2 + 3.0 * aA1; } +function B (aA1, aA2) { return 3.0 * aA2 - 6.0 * aA1; } +function C (aA1) { return 3.0 * aA1; } + +// Returns x(t) given t, x1, and x2, or y(t) given t, y1, and y2. +function calcBezier (aT, aA1, aA2) { return ((A(aA1, aA2) * aT + B(aA1, aA2)) * aT + C(aA1)) * aT; } + +// Returns dx/dt given t, x1, and x2, or dy/dt given t, y1, and y2. +function getSlope (aT, aA1, aA2) { return 3.0 * A(aA1, aA2) * aT * aT + 2.0 * B(aA1, aA2) * aT + C(aA1); } + +function binarySubdivide (aX, aA, aB, mX1, mX2) { + var currentX, currentT, i = 0; + do { + currentT = aA + (aB - aA) / 2.0; + currentX = calcBezier(currentT, mX1, mX2) - aX; + if (currentX > 0.0) { + aB = currentT; + } else { + aA = currentT; + } + } while (Math.abs(currentX) > SUBDIVISION_PRECISION && ++i < SUBDIVISION_MAX_ITERATIONS); + return currentT; +} + +function newtonRaphsonIterate (aX, aGuessT, mX1, mX2) { + for (var i = 0; i < NEWTON_ITERATIONS; ++i) { + var currentSlope = getSlope(aGuessT, mX1, mX2); + if (currentSlope === 0.0) { + return aGuessT; + } + var currentX = calcBezier(aGuessT, mX1, mX2) - aX; + aGuessT -= currentX / currentSlope; + } + return aGuessT; +} + +function LinearEasing (x) { + return x; +} + +module.exports = function bezier (mX1, mY1, mX2, mY2) { + if (!(0 <= mX1 && mX1 <= 1 && 0 <= mX2 && mX2 <= 1)) { + throw new Error('bezier x values must be in [0, 1] range'); + } + + if (mX1 === mY1 && mX2 === mY2) { + return LinearEasing; + } + + // Precompute samples table + var sampleValues = float32ArraySupported ? new Float32Array(kSplineTableSize) : new Array(kSplineTableSize); + for (var i = 0; i < kSplineTableSize; ++i) { + sampleValues[i] = calcBezier(i * kSampleStepSize, mX1, mX2); + } + + function getTForX (aX) { + var intervalStart = 0.0; + var currentSample = 1; + var lastSample = kSplineTableSize - 1; + + for (; currentSample !== lastSample && sampleValues[currentSample] <= aX; ++currentSample) { + intervalStart += kSampleStepSize; + } + --currentSample; + + // Interpolate to provide an initial guess for t + var dist = (aX - sampleValues[currentSample]) / (sampleValues[currentSample + 1] - sampleValues[currentSample]); + var guessForT = intervalStart + dist * kSampleStepSize; + + var initialSlope = getSlope(guessForT, mX1, mX2); + if (initialSlope >= NEWTON_MIN_SLOPE) { + return newtonRaphsonIterate(aX, guessForT, mX1, mX2); + } else if (initialSlope === 0.0) { + return guessForT; + } else { + return binarySubdivide(aX, intervalStart, intervalStart + kSampleStepSize, mX1, mX2); + } + } + + return function BezierEasing (x) { + // Because JavaScript number are imprecise, we should guarantee the extremes are right. + if (x === 0) { + return 0; + } + if (x === 1) { + return 1; + } + return calcBezier(getTForX(x), mY1, mY2); + }; +}; + +},{}],9:[function(require,module,exports){ +module.exports = function eventify(subject) { + validateSubject(subject); + + var eventsStorage = createEventsStorage(subject); + subject.on = eventsStorage.on; + subject.off = eventsStorage.off; + subject.fire = eventsStorage.fire; + return subject; +}; + +function createEventsStorage(subject) { + // Store all event listeners to this hash. Key is event name, value is array + // of callback records. + // + // A callback record consists of callback function and its optional context: + // { 'eventName' => [{callback: function, ctx: object}] } + var registeredEvents = Object.create(null); + + return { + on: function (eventName, callback, ctx) { + if (typeof callback !== 'function') { + throw new Error('callback is expected to be a function'); + } + var handlers = registeredEvents[eventName]; + if (!handlers) { + handlers = registeredEvents[eventName] = []; + } + handlers.push({callback: callback, ctx: ctx}); + + return subject; + }, + + off: function (eventName, callback) { + var wantToRemoveAll = (typeof eventName === 'undefined'); + if (wantToRemoveAll) { + // Killing old events storage should be enough in this case: + registeredEvents = Object.create(null); + return subject; + } + + if (registeredEvents[eventName]) { + var deleteAllCallbacksForEvent = (typeof callback !== 'function'); + if (deleteAllCallbacksForEvent) { + delete registeredEvents[eventName]; + } else { + var callbacks = registeredEvents[eventName]; + for (var i = 0; i < callbacks.length; ++i) { + if (callbacks[i].callback === callback) { + callbacks.splice(i, 1); + } + } + } + } + + return subject; + }, + + fire: function (eventName) { + var callbacks = registeredEvents[eventName]; + if (!callbacks) { + return subject; + } + + var fireArguments; + if (arguments.length > 1) { + fireArguments = Array.prototype.splice.call(arguments, 1); + } + for(var i = 0; i < callbacks.length; ++i) { + var callbackInfo = callbacks[i]; + callbackInfo.callback.apply(callbackInfo.ctx, fireArguments); + } + + return subject; + } + }; +} + +function validateSubject(subject) { + if (!subject) { + throw new Error('Eventify cannot use falsy object as events subject'); + } + var reservedWords = ['on', 'fire', 'off']; + for (var i = 0; i < reservedWords.length; ++i) { + if (subject.hasOwnProperty(reservedWords[i])) { + throw new Error("Subject cannot be eventified, since it already has property '" + reservedWords[i] + "'"); + } + } +} + +},{}],10:[function(require,module,exports){ +/** + * This module used to unify mouse wheel behavior between different browsers in 2014 + * Now it's just a wrapper around addEventListener('wheel'); + * + * Usage: + * var addWheelListener = require('wheel').addWheelListener; + * var removeWheelListener = require('wheel').removeWheelListener; + * addWheelListener(domElement, function (e) { + * // mouse wheel event + * }); + * removeWheelListener(domElement, function); + */ + +module.exports = addWheelListener; + +// But also expose "advanced" api with unsubscribe: +module.exports.addWheelListener = addWheelListener; +module.exports.removeWheelListener = removeWheelListener; + + +function addWheelListener(element, listener, useCapture) { + element.addEventListener('wheel', listener, useCapture); +} + +function removeWheelListener( element, listener, useCapture ) { + element.removeEventListener('wheel', listener, useCapture); +} +},{}]},{},[1])(1) +}); diff --git a/javascript/style.css b/javascript/style.css index 19faaa021..3487c461a 100644 --- a/javascript/style.css +++ b/javascript/style.css @@ -28,7 +28,7 @@ div.gradio-html.min{ min-height: 0; } footer { display: none; } /* general styled components */ -.gradio-button.tool{ max-width: 2.3em; min-width: 2.3em !important; height: 2.3em; align-self: end; line-height: 1em; border-radius: 0.5em; } +.gradio-button.tool{ max-width: 2.3em; min-width: 2.3em !important; height: 2.3em; align-self: end; line-height: 1em } .gradio-button.secondary-down{ background: var(--button-secondary-background-fill); color: var(--button-secondary-text-color); } .gradio-button.secondary-down, .gradio-button.secondary-down:hover{ box-shadow: 1px 1px 1px rgba(0,0,0,0.25) inset, 0px 0px 3px rgba(0,0,0,0.15) inset; } .gradio-button.secondary-down:hover{ background: var(--button-secondary-background-fill-hover); color: var(--button-secondary-text-color-hover); } @@ -69,7 +69,6 @@ button.custom-button{ background: var(--input-background-fill) !important; box-shadow: 0 0 0.0 0.3em rgba(192,192,192,0.15), inset 0 0 0.6em rgba(192,192,192,0.075); border: 2px solid rgba(192,192,192,0.4) !important; - border-radius: 0.4em; } .block.token-counter.error span{ @@ -100,7 +99,8 @@ button.custom-button{ } #txt2img_gallery img, #img2img_gallery img, #extras_gallery img { object-fit: scale-down; width: -webkit-fill-available !important; } -#txt2img_footer, #img2img_footer, #extras_footer{ height: fit-content; display: none; } +#txt2img_footer, #img2img_footer, #extras_footer { height: fit-content; } +#txt2img_footer, #img2img_footer { height: fit-content; display: none; } #txt2img_generate_box, #img2img_generate_box { gap: 0.5em; flex-wrap: wrap-reverse; } #txt2img_actions_column, #img2img_actions_column { gap: 0.5em; } #txt2img_generate_box > button, #img2img_generate_box > button { height: 2.2em; line-height: 0; } @@ -315,7 +315,6 @@ div#extras_scale_to_tab div.form{ position: relative; height: 20px; background: #b4c0cc; - border-radius: 3px !important; margin-bottom: -3px; } @@ -332,7 +331,6 @@ div#extras_scale_to_tab div.form{ line-height: 20px; padding: 0 8px 0 0; text-align: right; - border-radius: 3px; overflow: visible; white-space: nowrap; padding: 0 0.5em; @@ -434,6 +432,7 @@ div#extras_scale_to_tab div.form{ height: 100%; width: 100%; min-height: 0; + background: transparent; } table.settings-value-table{ @@ -463,14 +462,12 @@ table.settings-value-table td{ font-weight: bold; font-size: 20px; transition: 0.6s ease; - border-radius: 0 3px 3px 0; user-select: none; -webkit-user-select: none; } .modalNext { right: 0; - border-radius: 3px 0 0 3px; } .modalPrev:hover, .modalNext:hover { @@ -496,7 +493,6 @@ table.settings-value-table td{ display:block; padding:0px 0; border:2px solid #a55000; - border-radius:8px; box-shadow:1px 1px 2px #CE6400; width: 200px; } @@ -538,11 +534,11 @@ table.settings-value-table td{ .extra-networks .description { margin-top: 8px; } .extra-networks .tab-nav > button { margin-right: 0; height: auto; padding: 2px 4px 2px 4px; } .extra-networks-tab { padding: 0 !important; } -.extra-network-subdirs { background: var(--input-background-fill); } +.extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; min-width: 80px; max-width: 120px; } .extra-networks-page { display: flex } -.extra-networks .custom-button { min-width: 60px; width: 100%; background: none; justify-content: left; padding: 2px 8px 2px 8px; box-shadow: none; } +.extra-networks .custom-button { min-width: 80px; max-width: 120px; width: 100%; background: none; justify-content: left; text-align: left; padding: 2px 8px 2px 8px; box-shadow: none; line-break: auto; } .extra-networks .custom-button:hover { background: var(--button-primary-background-fill) } -.extra-network-cards { display: flex; flex-wrap: wrap; height: 50vh; overflow-y: scroll; overflow-x: hidden; scroll-snap-type: y mandatory; width: -webkit-fill-available; } +.extra-network-cards { display: flex; flex-wrap: wrap; overflow-y: scroll; overflow-x: hidden; width: -webkit-fill-available; } .extra-network-cards .card { height: fit-content; margin: 0.5em; position: relative; scroll-snap-align: start; scroll-margin-top: 0; } .extra-network-cards .card .overlay { position: absolute; bottom: 0; padding: 0.2em; z-index: 10; width: 100%; background: none; } .extra-network-cards .card:hover .overlay { background: rgba(0, 0, 0, 0.40); } diff --git a/launch.py b/launch.py index aea1b404b..cac19f277 100644 --- a/launch.py +++ b/launch.py @@ -174,6 +174,7 @@ if __name__ == "__main__": installer.install_repositories() installer.install_submodules() installer.install_extensions() + installer.install_packages() # redo packages since extensions may change them installer.update_wiki() if installer.errors == 0: installer.log.debug(f'Setup complete without errors: {round(time.time())}') diff --git a/modules/api/api.py b/modules/api/api.py index 82853117f..eda9612a4 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -237,7 +237,7 @@ class Api: script_runner = scripts.scripts_txt2img if not script_runner.scripts: script_runner.initialize_scripts(False) - ui.create_ui() + ui.create_ui(None) if not self.default_script_arg_txt2img: self.default_script_arg_txt2img = self.init_default_script_args(script_runner) selectable_scripts, selectable_script_idx = self.get_selectable_script(txt2imgreq.script_name, script_runner) @@ -282,7 +282,7 @@ class Api: script_runner = scripts.scripts_img2img if not script_runner.scripts: script_runner.initialize_scripts(True) - ui.create_ui() + ui.create_ui(None) if not self.default_script_arg_img2img: self.default_script_arg_img2img = self.init_default_script_args(script_runner) selectable_scripts, selectable_script_idx = self.get_selectable_script(img2imgreq.script_name, script_runner) @@ -406,17 +406,15 @@ class Api: def interruptapi(self): shared.state.interrupt() - return {} def unloadapi(self): - unload_model_weights() - + unload_model_weights(op='model') + unload_model_weights(op='refiner') return {} def reloadapi(self): reload_model_weights() - return {} def skip(self): @@ -650,8 +648,8 @@ class Api: "port": shared.cmd_opts.port, "keyfile": shared.cmd_opts.tls_keyfile, "certfile": shared.cmd_opts.tls_certfile, - "loop": "auto", - "http": "auto", + "loop": "auto", # auto, asyncio, uvloop + "http": "auto", # auto, h11, httptools } from modules.server import UvicornServer server = UvicornServer(self.app, **config) diff --git a/modules/api/models.py b/modules/api/models.py index 1b3399f18..704ebfe92 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -1,7 +1,6 @@ import inspect from typing import Any, Optional, Dict, List from pydantic import BaseModel, Field, create_model # pylint: disable=no-name-in-module -from typing_extensions import Literal from inflection import underscore from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img import modules.shared as shared @@ -133,7 +132,7 @@ class ImageToImageResponse(BaseModel): info: str class ExtrasBaseRequest(BaseModel): - resize_mode: Literal[0, 1] = Field(default=0, title="Resize Mode", description="Sets the resize mode: 0 to upscale by upscaling_resize amount, 1 to upscale up to upscaling_resize_h x upscaling_resize_w.") + resize_mode: float = Field(default=0, title="Resize Mode", description="Sets the resize mode: 0 to upscale by upscaling_resize amount, 1 to upscale up to upscaling_resize_h x upscaling_resize_w.") show_extras_results: bool = Field(default=True, title="Show results", description="Should the backend return the generated image?") gfpgan_visibility: float = Field(default=0, title="GFPGAN Visibility", ge=0, le=1, allow_inf_nan=False, description="Sets the visibility of GFPGAN, values should be between 0 and 1.") codeformer_visibility: float = Field(default=0, title="CodeFormer Visibility", ge=0, le=1, allow_inf_nan=False, description="Sets the visibility of CodeFormer, values should be between 0 and 1.") diff --git a/modules/cmd_args.py b/modules/cmd_args.py index 3c0c2ee6c..fe37814e9 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -45,7 +45,7 @@ group.add_argument('--use-directml', default = False, action='store_true', help group.add_argument("--use-cuda", default=False, action='store_true', help="Force use nVidia CUDA backend, default: %(default)s") group.add_argument("--use-rocm", default=False, action='store_true', help="Force use AMD ROCm backend, default: %(default)s") group.add_argument('--subpath', type=str, help='Customize the URL subpath for usage with reverse proxy') -group.add_argument('--backend', type=str, choices=[None, 'original', 'diffusers'], default=None, required=False, help='force backend type') +group.add_argument('--backend', type=str, choices=['original', 'diffusers'], required=False, help='force model pipeline type') # removed args are added here as hidden in fixed format for compatbility reasons @@ -90,6 +90,8 @@ def compatibility_args(opts, args): group.add_argument("--sub-quad-chunk-threshold", help=argparse.SUPPRESS, default=opts.sub_quad_chunk_threshold) group.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir) group.add_argument("--lyco-dir", help=argparse.SUPPRESS, default=opts.lyco_dir) + group.add_argument("--embeddings-dir", help=argparse.SUPPRESS, default=opts.embeddings_dir) + group.add_argument("--hypernetwork-dir", help=argparse.SUPPRESS, default=opts.hypernetwork_dir) group.add_argument("--lyco-patch-lora", help=argparse.SUPPRESS, default=opts.lyco_patch_lora) group.add_argument("--lyco-debug", help=argparse.SUPPRESS, action='store_true', default=False) group.add_argument("--enable-console-prompts", help=argparse.SUPPRESS, action='store_true', default=False) diff --git a/modules/devices.py b/modules/devices.py index dbc7b94a7..009f45fb1 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -8,6 +8,8 @@ if sys.platform == "darwin": from modules import mac_specific # pylint: disable=ungrouped-imports cuda_ok = torch.cuda.is_available() +previous_oom = 0 + def has_mps() -> bool: if sys.platform != "darwin": @@ -15,6 +17,7 @@ def has_mps() -> bool: else: return mac_specific.has_mps + def extract_device_id(args, name): # pylint: disable=redefined-outer-name for x in range(len(args)): if name in args[x]: @@ -61,6 +64,18 @@ def get_device_for(task): def torch_gc(force=False): + mem = memstats.memory_stats() + gpu = mem.get('gpu', {}) + oom = gpu.get('oom', 0) + used = round(100 * gpu.get('used', 0) / gpu.get('total', 1)) + global previous_oom # pylint: disable=global-statement + if oom > previous_oom: + previous_oom = oom + shared.log.warning(f'GPU out-of-memory error: {mem}') + if used > 90: + shared.log.warning(f'GPU high memory utilization: {used}% {mem}') + force = True + if shared.opts.disable_gc and not force: return collected = gc.collect() @@ -122,6 +137,7 @@ def set_cuda_params(): try: torch.backends.cudnn.benchmark = True if shared.opts.cudnn_benchmark: + shared.log.debug('Torch enable cuDNN benchmark') torch.backends.cudnn.benchmark_limit = 0 torch.backends.cudnn.allow_tf32 = shared.opts.cuda_allow_tf32 except Exception: diff --git a/modules/dml/__init__.py b/modules/dml/__init__.py index 6d38e9f88..26639c428 100644 --- a/modules/dml/__init__.py +++ b/modules/dml/__init__.py @@ -3,6 +3,7 @@ import torch import torch_directml # pylint: disable=import-error import modules.dml.hijack import modules.dml.amp as amp +from modules.dml.opts import override_opts from .optimizer.unknown import UnknownOptimizer @@ -31,6 +32,8 @@ class DirectML(): return optimizer.memory_stats(device.index) -# Alternative of torch.cuda for DirectML. DirectML.amp = amp +# Alternative of torch.cuda for DirectML. torch.dml = DirectML + +override_opts() diff --git a/modules/dml/hijack/__init__.py b/modules/dml/hijack/__init__.py index d8cc0913a..7e3424775 100644 --- a/modules/dml/hijack/__init__.py +++ b/modules/dml/hijack/__init__.py @@ -3,3 +3,4 @@ import modules.dml.hijack.stablediffusion import modules.dml.hijack.torch import modules.dml.hijack.realesrgan_model import modules.dml.hijack.plms +import modules.dml.hijack.diffusers diff --git a/modules/dml/hijack/diffusers.py b/modules/dml/hijack/diffusers.py new file mode 100644 index 000000000..3279398cf --- /dev/null +++ b/modules/dml/hijack/diffusers.py @@ -0,0 +1,48 @@ +from torch import FloatTensor +import diffusers + +def _get_prev_sample(self, sample: FloatTensor, timestep, prev_timestep, model_output): + # See formula (9) of PNDM paper https://arxiv.org/pdf/2202.09778.pdf + # this function computes x_(t−δ) using the formula of (9) + # Note that x_t needs to be added to both sides of the equation + + # Notation ( -> + # alpha_prod_t -> α_t + # alpha_prod_t_prev -> α_(t−δ) + # beta_prod_t -> (1 - α_t) + # beta_prod_t_prev -> (1 - α_(t−δ)) + # sample -> x_t + # model_output -> e_θ(x_t, t) + # prev_sample -> x_(t−δ) + sample.__str__() # DML Solution: PNDM Sampling does not work without 'stringify'. (because it depends on PLMS) + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + if self.config.prediction_type == "v_prediction": + model_output = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + elif self.config.prediction_type != "epsilon": + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon` or `v_prediction`" + ) + + # corresponds to (α_(t−δ) - α_t) divided by + # denominator of x_t in formula (9) and plus 1 + # Note: (α_(t−δ) - α_t) / (sqrt(α_t) * (sqrt(α_(t−δ)) + sqr(α_t))) = + # sqrt(α_(t−δ)) / sqrt(α_t)) + sample_coeff = (alpha_prod_t_prev / alpha_prod_t) ** (0.5) + + # corresponds to denominator of e_θ(x_t, t) in formula (9) + model_output_denom_coeff = alpha_prod_t * beta_prod_t_prev ** (0.5) + ( + alpha_prod_t * beta_prod_t * alpha_prod_t_prev + ) ** (0.5) + + # full formula (9) + prev_sample = ( + sample_coeff * sample - (alpha_prod_t_prev - alpha_prod_t) * model_output / model_output_denom_coeff + ) + + return prev_sample + +diffusers.PNDMScheduler._get_prev_sample = _get_prev_sample diff --git a/modules/dml/hijack/plms.py b/modules/dml/hijack/plms.py index 5a9570454..2baef815d 100644 --- a/modules/dml/hijack/plms.py +++ b/modules/dml/hijack/plms.py @@ -48,7 +48,7 @@ def p_sample_plms(self, x, c, t, index, repeat_noise=False, use_original_steps=F def get_x_prev_and_pred_x0(e_t, index): # select parameters corresponding to the currently considered timestep - print(alphas[index]) # DML Solution: PLMS Sampling does not work without this print. + alphas[index].__str__() # DML Solution: PLMS Sampling does not work without this 'stringify'. a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) diff --git a/modules/dml/hijack/stablediffusion.py b/modules/dml/hijack/stablediffusion.py index cbc4b85fb..b14b0ece1 100644 --- a/modules/dml/hijack/stablediffusion.py +++ b/modules/dml/hijack/stablediffusion.py @@ -51,7 +51,7 @@ def p_sample_ddim(self, x, c, t, index, repeat_noise=False, use_original_steps=F sqrt_one_minus_alphas = self.model.sqrt_one_minus_alphas_cumprod if use_original_steps else self.ddim_sqrt_one_minus_alphas sigmas = self.model.ddim_sigmas_for_original_num_steps if use_original_steps else self.ddim_sigmas # select parameters corresponding to the currently considered timestep - print(alphas[index]) # DML Solution: DDIM Sampling does not work without this print. + alphas[index].__str__() # DML Solution: DDIM Sampling does not work without this 'stringify'. a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) diff --git a/modules/dml/opts.py b/modules/dml/opts.py new file mode 100644 index 000000000..59fe392a0 --- /dev/null +++ b/modules/dml/opts.py @@ -0,0 +1,5 @@ +from modules import shared + +def override_opts(): + if shared.backend == shared.Backend.DIFFUSERS: + shared.opts.diffusers_generator_device = "cpu" # DirectML does not support torch.Generator API. diff --git a/modules/images.py b/modules/images.py index efa1f7a3d..e0c8b4bbb 100644 --- a/modules/images.py +++ b/modules/images.py @@ -133,7 +133,7 @@ def draw_grid_annotations(im, width, height, hor_texts, ver_texts, margin=0): except Exception: return ImageFont.truetype('html/roboto.ttf', fontsize) - def draw_texts(drawing, draw_x, draw_y, lines, initial_fnt, initial_fontsize): + def draw_texts(drawing: ImageDraw, draw_x, draw_y, lines, initial_fnt, initial_fontsize): for line in lines: fnt = initial_fnt fontsize = initial_fontsize diff --git a/modules/lora_diffusers.py b/modules/lora_diffusers.py new file mode 100644 index 000000000..e5779b3c6 --- /dev/null +++ b/modules/lora_diffusers.py @@ -0,0 +1,34 @@ +import diffusers +from modules import shared + +lora_state = { # TODO Lora state for Diffusers + 'multiplier': 1.0, + 'active': False, + 'loaded': 0, +} + +def unload_diffusers_lora(): + try: + pipe = shared.sd_model + lora_state['active'] = False + lora_state['loaded'] = 0 + pipe._remove_text_encoder_monkey_patch() # pylint: disable=W0212 + proc_cls_name = next(iter(pipe.unet.attn_processors.values())).__class__.__name__ + non_lora_proc_cls = getattr(diffusers.models.attention_processor, proc_cls_name[len("LORA"):]) + pipe.unet.set_attn_processor(non_lora_proc_cls()) + # shared.log.debug('Diffusers LoRA unloaded') + except Exception: + pass + + +def load_diffusers_lora(name, lora, strength = 1.0): + try: + pipe = shared.sd_model + pipe.load_lora_weights(lora.filename, cache_dir=shared.opts.diffusers_dir, local_files_only=True) + lora_state['active'] = True + lora_state['loaded'] += 1 + lora_state['multiplier'] = strength + # pipe.unet.load_attn_procs("pcuenq/pokemon-lora") + shared.log.info(f"Diffusers LoRA loaded: {name} {lora_state['multiplier']}") + except Exception as e: + shared.log.error(f"Diffusers LoRA loading failed: {name} {e}") diff --git a/modules/modelloader.py b/modules/modelloader.py index c49cdeb5d..a2d616b01 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -1,6 +1,8 @@ import os import shutil import importlib +import json +from typing import Dict from urllib.parse import urlparse from modules import shared @@ -9,30 +11,60 @@ from modules.paths import script_path, models_path diffuser_repos = [] -def load_diffusers(model_path: str, hub_url: str = None, command_path: str = None): - import huggingface_hub as hf + +def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config: Dict[str, str] = None, token = None): from diffusers import DiffusionPipeline + import huggingface_hub as hf + if download_config is None: + download_config = { + "force_download": False, + "resume_download": True, + "cache_dir": shared.opts.diffusers_dir, + # "use_auth_token": True, + } + if cache_dir is not None: + download_config["cache_dir"] = cache_dir + shared.log.debug(f"Diffusers downloading: {hub_id} to {cache_dir}") + if token is not None and len(token) > 2: + shared.log.debug(f"Diffusers authentication: {token}") + hf.login(token) + pipeline_dir = DiffusionPipeline.download(hub_id, **download_config) + try: + model_info_dict = hf.model_info(hub_id).cardData # TODO HF-Hub cardData invalid property + except Exception: + model_info_dict = None + # some checkpoints need to be downloaded as "hidden" as they just serve as pre- or post-pipelines of other pipelines + if model_info_dict is not None and "prior" in model_info_dict: + download_dir = DiffusionPipeline.download(model_info_dict["prior"], **download_config) + model_info_dict["prior"] = download_dir + # mark prior as hidden + with open(os.path.join(download_dir, "hidden"), "w", encoding="utf-8") as f: + f.write("True") + with open(os.path.join(pipeline_dir, "model_info.json"), "w", encoding="utf-8") as json_file: + json.dump(model_info_dict, json_file) + return pipeline_dir + + +def load_diffusers_models(model_path: str, command_path: str = None): + import huggingface_hub as hf places = [] - - # download repo - if hub_url is not None: - DiffusionPipeline.download(hub_url, cache_dir=model_path) - places.append(model_path) if command_path is not None and command_path != model_path and os.path.isdir(command_path): places.append(command_path) diffuser_repos.clear() output = [] - try: - for place in places: + for place in places: + try: res = hf.scan_cache_dir(cache_dir=place) for r in list(res.repos): - diffuser_repos.append({ 'name': r.repo_id, 'filename': r.repo_id, 'path': str(r.repo_path), 'size': r.size_on_disk, 'mtime': r.last_modified, 'hash': list(r.revisions)[-1].commit_hash }) - output.append(str(r.repo_id)) - except Exception as e: - shared.log.error(f"Error listing diffusers: {place} {e}") - shared.log.debug(f'Scanning diffusers cache: {len(output)} {model_path} {command_path}') + cache_path = os.path.join(r.repo_path, "snapshots", list(r.revisions)[-1].commit_hash) + diffuser_repos.append({ 'name': r.repo_id, 'filename': r.repo_id, 'path': cache_path, 'size': r.size_on_disk, 'mtime': r.last_modified, 'hash': list(r.revisions)[-1].commit_hash, 'model_info': str(os.path.join(cache_path, "model_info.json")) }) + if not os.path.isfile(os.path.join(cache_path, "hidden")): + output.append(str(r.repo_id)) + except Exception as e: + shared.log.error(f"Error listing diffusers: {place} {e}") + shared.log.debug(f'Scanning diffusers cache: {model_path} {command_path} {len(output)}') return output @@ -47,10 +79,9 @@ def find_diffuser(name: str): filt = hf.ModelFilter( model_name=name, task='text-to-image', - tags='stable-diffusion', - library=['diffusers', 'stable-diffusion'], + library=['diffusers'], ) - models = list(api.list_models(filter=filt, full=True, limit=50, sort="downloads", direction=-1)) + models = list(api.list_models(filter=filt, full=True, limit=5, sort="downloads", direction=-1)) shared.log.debug(f'Searching diffusers models: {name} {len(models) > 0}') if len(models) > 0: return models[0].modelId @@ -77,7 +108,7 @@ def load_models(model_path: str, model_url: str = None, command_path: str = None for place in places: for full_path in shared.walk_files(place, allowed_extensions=ext_filter): if os.path.islink(full_path) and not os.path.exists(full_path): - print(f"Skipping broken symlink: {full_path}") + shared.log.error(f"Skipping broken symlink: {full_path}") continue if ext_blacklist is not None and any(full_path.endswith(x) for x in ext_blacklist): continue @@ -144,13 +175,13 @@ def move_files(src_path: str, dest_path: str, ext_filter: str = None): if ext_filter is not None: if ext_filter not in file: continue - print(f"Moving {file} from {src_path} to {dest_path}.") + shared.log.warning(f"Moving {file} from {src_path} to {dest_path}.") try: shutil.move(fullpath, dest_path) except Exception: pass if len(os.listdir(src_path)) == 0: - print(f"Removing empty folder: {src_path}") + shared.log.info(f"Removing empty folder: {src_path}") shutil.rmtree(src_path, True) except Exception: pass diff --git a/modules/models/diffusion/ddpm_edit.py b/modules/models/diffusion/ddpm_edit.py index c847bfd68..ad067dd8b 100644 --- a/modules/models/diffusion/ddpm_edit.py +++ b/modules/models/diffusion/ddpm_edit.py @@ -174,7 +174,6 @@ class DDPM(pl.LightningModule): lvlb_weights = 0.5 * np.sqrt(torch.Tensor(alphas_cumprod)) / (2. * 1 - torch.Tensor(alphas_cumprod)) else: raise NotImplementedError("mu not supported") - # TODO how to choose this term lvlb_weights[0] = lvlb_weights[1] self.register_buffer('lvlb_weights', lvlb_weights, persistent=False) assert not torch.isnan(self.lvlb_weights).all() @@ -631,7 +630,7 @@ class LatentDiffusion(DDPM): weighting = weighting * L_weighting return weighting - def get_fold_unfold(self, x, kernel_size, stride, uf=1, df=1): # todo load once not every time, shorten code + def get_fold_unfold(self, x, kernel_size, stride, uf=1, df=1): """ :param x: img of size (bs, c, h, w) :return: n img crops of size (n, bs, c, kernel_size[0], kernel_size[1]) @@ -887,7 +886,7 @@ class LatentDiffusion(DDPM): assert c is not None if self.cond_stage_trainable: c = self.get_learned_conditioning(c) - if self.shorten_cond_schedule: # TODO: drop this option + if self.shorten_cond_schedule: tc = self.cond_ids[t].to(self.device) c = self.q_sample(x_start=c, t=tc, noise=torch.randn_like(c.float())) return self.p_losses(x, c, t, *args, **kwargs) @@ -919,7 +918,7 @@ class LatentDiffusion(DDPM): z_list = [z[:, :, :, :, i] for i in range(z.shape[-1])] if self.cond_stage_key in ["image", "LR_image", "segmentation", - 'bbox_img'] and self.model.conditioning_key: # todo check for completeness + 'bbox_img'] and self.model.conditioning_key: c_key = next(iter(cond.keys())) # get key c = next(iter(cond.values())) # get value assert (len(c) == 1) # todo extend to list with more than one elem @@ -973,12 +972,11 @@ class LatentDiffusion(DDPM): cond_list = [{'c_crossattn': [e]} for e in adapted_cond] else: - cond_list = [cond for i in range(z.shape[-1])] # Todo make this more efficient + cond_list = [cond for i in range(z.shape[-1])] # apply model by loop over crops output_list = [self.model(z_list[i], t, **cond_list[i]) for i in range(z.shape[-1])] - assert not isinstance(output_list[0], - tuple) # todo cant deal with multiple model outputs check this never happens + assert not isinstance(output_list[0], tuple) o = torch.stack(output_list, axis=-1) o = o * weighting @@ -1431,7 +1429,6 @@ class DiffusionWrapper(pl.LightningModule): class Layout2ImgDiffusion(LatentDiffusion): - # TODO: move all layout-specific hacks to this class def __init__(self, cond_stage_key, *args, **kwargs): assert cond_stage_key == 'coordinates_bbox', 'Layout2ImgDiffusion only for cond_stage_key="coordinates_bbox"' super().__init__(*args, cond_stage_key=cond_stage_key, **kwargs) diff --git a/modules/paths.py b/modules/paths.py index deb022c75..966349580 100644 --- a/modules/paths.py +++ b/modules/paths.py @@ -76,6 +76,7 @@ def create_paths(opts): create_path(fix_path('hypernetwork_dir')) create_path(fix_path('ckpt_dir')) create_path(fix_path('vae_dir')) + create_path(fix_path('diffusers_dir')) create_path(fix_path('embeddings_dir')) create_path(fix_path('outdir_samples')) create_path(fix_path('outdir_txt2img_samples')) diff --git a/modules/processing.py b/modules/processing.py index 6d05beed4..4854a1c31 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -26,6 +26,7 @@ import modules.images as images import modules.styles import modules.sd_models as sd_models import modules.sd_vae as sd_vae +from modules.lora_diffusers import lora_state, unload_diffusers_lora opt_C = 4 @@ -222,8 +223,9 @@ class StableDiffusionProcessing: source_image = devices.cond_cast_float(source_image) # HACK: Using introspection as the Depth2Image model doesn't appear to uniquely # identify itself with a field common to all models. The conditioning_key is also hybrid. - if backend == Backend.DIFFUSERS: # TODO: Diffusers img2img_image_conditioning - return latent_image.new_zeros(latent_image.shape[0], 5, 1, 1) + if backend == Backend.DIFFUSERS: + log.warning('Diffusers not implemented: img2img_image_conditioning') + return None if isinstance(self.sd_model, LatentDepth2ImageDiffusion): return self.depth2img_image_conditioning(source_image) if self.sd_model.cond_stage_key == "edit": @@ -445,24 +447,23 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_su if uses_ensd: uses_ensd = sd_samplers_common.is_sampler_using_eta_noise_seed_delta(p) - generation_params = { "Steps": p.steps, "Sampler": p.sampler_name, "CFG scale": p.cfg_scale, "Image CFG scale": getattr(p, 'image_cfg_scale', None), "Seed": all_seeds[index], - "Face restoration": (opts.face_restoration_model if p.restore_faces else None), + "Face restoration": opts.face_restoration_model if p.restore_faces else None, "Size": f"{p.width}x{p.height}", "Model hash": getattr(p, 'sd_model_hash', None if not opts.add_model_hash_to_info or not shared.sd_model.sd_model_hash else shared.sd_model.sd_model_hash), - "Model": (None if not opts.add_model_name_to_info or not shared.sd_model.sd_checkpoint_info.model_name else shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', '')), - "VAE": (None if not opts.add_model_name_to_info or sd_vae.loaded_vae_file is None else os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0]), - "Variation seed": (None if p.subseed_strength == 0 else all_subseeds[index]), - "Variation seed strength": (None if p.subseed_strength == 0 else p.subseed_strength), - "Seed resize from": (None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}"), + "Model": None if not opts.add_model_name_to_info or not shared.sd_model.sd_checkpoint_info.model_name else shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', ''), + "VAE": None if not opts.add_model_name_to_info or sd_vae.loaded_vae_file is None else os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0], + "Variation seed": None if p.subseed_strength == 0 else all_subseeds[index], + "Variation seed strength": None if p.subseed_strength == 0 else p.subseed_strength, + "Seed resize from": None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}", "Denoising strength": getattr(p, 'denoising_strength', None), "Conditional mask weight": getattr(p, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) if p.is_using_inpainting_conditioning else None, - "Clip skip": p.clip_skip, + "Clip skip": p.clip_skip if p.clip_skip > 1 else None, "ENSD": opts.eta_noise_seed_delta if uses_ensd else None, "Init image hash": getattr(p, 'init_img_hash', None), "Version": git_commit, @@ -520,7 +521,8 @@ def process_images(p: StableDiffusionProcessing) -> Processed: if k == 'sd_vae': sd_vae.reload_vae_weights() - sd_models.apply_token_merging(p.sd_model, p.get_token_merging_ratio()) + if not shared.opts.cuda_compile: + sd_models.apply_token_merging(p.sd_model, p.get_token_merging_ratio()) if cmd_opts.profile: """ @@ -538,7 +540,8 @@ def process_images(p: StableDiffusionProcessing) -> Processed: else: res = process_images_inner(p) finally: - sd_models.apply_token_merging(p.sd_model, 0) + if not shared.opts.cuda_compile: + sd_models.apply_token_merging(p.sd_model, 0) if p.override_settings_restore_afterwards: # restore opts to original state for k, v in stored_opts.items(): setattr(opts, k, v) @@ -557,6 +560,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: assert len(p.prompt) > 0 else: assert p.prompt is not None + seed = get_fixed_seed(p.seed) subseed = get_fixed_seed(p.subseed) if backend == Backend.ORIGINAL: @@ -678,31 +682,91 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: x_samples_ddim = torch.stack(x_samples_ddim).float() x_samples_ddim = torch.clamp((x_samples_ddim + 1.0) / 2.0, min=0.0, max=1.0) del samples_ddim - if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: - lowvram.send_everything_to_cpu() - devices.torch_gc() - if p.scripts is not None: - p.scripts.postprocess_batch(p, x_samples_ddim, batch_number=n) - else: # TODO Diffusers main processing - generator = [torch.Generator(device="cpu").manual_seed(s) for s in seeds] - if shared.sd_model.scheduler.name != p.sampler_name: + + elif backend == Backend.DIFFUSERS: + generator_device = 'cpu' if shared.opts.diffusers_generator_device == "cpu" else shared.device + generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds] + if (not hasattr(shared.sd_model.scheduler, 'name')) or (shared.sd_model.scheduler.name != p.sampler_name): sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") - scheduler = sampler.constructor(shared.sd_model.sd_checkpoint_info.filename) - shared.sd_model.scheduler = scheduler.sampler - output = shared.sd_model( + shared.sd_model.scheduler = sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op + + cross_attention_kwargs={} + if lora_state['active']: + cross_attention_kwargs['scale'] = lora_state['multiplier'] + task_specific_kwargs={} + if sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE: + task_specific_kwargs = {"height": p.height, "width": p.width} + elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE: + task_specific_kwargs = {"image": p.init_images[0], "strength": p.denoising_strength} + elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING: + # TODO(PVP): change out to latents once possible with `diffusers` + task_specific_kwargs = {"image": p.init_images[0], "mask_image": p.image_mask, "strength": p.denoising_strength} + + # TODO Diffusers limited callbacks + # TODO Diffusers processing is not using p.sample so second pass is ignored + def diffusers_callback(step: int, _timestep: int, latents: torch.FloatTensor): + shared.state.sampling_step = step + shared.state.sampling_steps = p.steps + shared.state.current_latent = latents + shared.state.set_current_image() + + # shared.sd_model.to(devices.device) + output = shared.sd_model( # pylint: disable=not-callable prompt=prompts, negative_prompt=negative_prompts, num_inference_steps=p.steps, guidance_scale=p.cfg_scale, - height=p.height, - width=p.width, generator=generator, - output_type="np", + callback_steps = 1, + callback = diffusers_callback, + output_type='np' if shared.sd_refiner is None else 'latent', + cross_attention_kwargs=cross_attention_kwargs, + **task_specific_kwargs ) + # if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: + # shared.sd_model.to('cpu') + # devices.torch_gc(force=True) + + if shared.sd_refiner is not None: + # shared.sd_refiner.to(devices.device) + devices.torch_gc() + init_image = output.images[0] + output = shared.sd_refiner( # pylint: disable=not-callable + prompt=prompts, + negative_prompt=negative_prompts, + num_inference_steps=p.steps, + guidance_scale=p.cfg_scale, + generator=generator, + callback_steps = 1, + callback = diffusers_callback, + output_type='np', + cross_attention_kwargs=cross_attention_kwargs, + image=init_image + ) + # if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: + # shared.sd_refiner.to('cpu') + # devices.torch_gc(force=True) + + x_samples_ddim = output.images + if p.enable_hr: + log.warning('Diffusers not implemented: hires fix') + + if lora_state['active']: + unload_diffusers_lora() + + else: + raise ValueError(f"Unknown backend {backend}") + + if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: + lowvram.send_everything_to_cpu() + devices.torch_gc() + if p.scripts is not None: + p.scripts.postprocess_batch(p, x_samples_ddim, batch_number=n) + for i, x_sample in enumerate(x_samples_ddim): p.batch_index = i if backend == Backend.ORIGINAL: @@ -820,8 +884,12 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.applied_old_hires_behavior_to = None def init(self, all_prompts, all_seeds, all_subseeds): + if backend == Backend.DIFFUSERS: + sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) + self.width = self.width or 512 self.height = self.height or 512 + if self.enable_hr: if opts.use_old_hires_fix_width_height and self.applied_old_hires_behavior_to != (self.width, self.height): self.hr_resize_x = self.width @@ -872,20 +940,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): if self.hr_upscaler is not None: self.extra_generation_params["Hires upscaler"] = self.hr_upscaler - def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): # TODO this is majority of processing time - self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model) - latent_scale_mode = shared.latent_upscale_modes.get(self.hr_upscaler, None) if self.hr_upscaler is not None else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "nearest") - if self.enable_hr and latent_scale_mode is None: - if len([x for x in shared.sd_upscalers if x.name == self.hr_upscaler]) == 0: - log.warning("Could not find upscaler to use with hrfix") - self.enable_hr = False - x = create_random_tensors([opt_C, self.height // opt_f, self.width // opt_f], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self) - samples = self.sampler.sample(self, x, conditioning, unconditional_conditioning, image_conditioning=self.txt2img_image_conditioning(x)) - if not self.enable_hr or state.interrupted or state.skipped: - return samples - self.is_hr_pass = True - target_width = self.hr_upscale_to_x - target_height = self.hr_upscale_to_y + def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): def save_intermediate(image, index): """saves image before applying hires fix, if enabled in options; takes as an argument either an image or batch with latent space images""" @@ -902,6 +957,23 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.restore_faces = orig2 images.save_image(image, self.outpath_samples, "", seeds[index], prompts[index], opts.samples_format, info=info, suffix="-before-highres-fix") + if backend == Backend.DIFFUSERS: + sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) + + self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model) + latent_scale_mode = shared.latent_upscale_modes.get(self.hr_upscaler, None) if self.hr_upscaler is not None else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "nearest") + if self.enable_hr and latent_scale_mode is None: + if len([x for x in shared.sd_upscalers if x.name == self.hr_upscaler]) == 0: + log.warning("Could not find upscaler to use with hrfix") + self.enable_hr = False + x = create_random_tensors([opt_C, self.height // opt_f, self.width // opt_f], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self) + samples = self.sampler.sample(self, x, conditioning, unconditional_conditioning, image_conditioning=self.txt2img_image_conditioning(x)) + if not self.enable_hr or state.interrupted or state.skipped: + return samples + self.is_hr_pass = True + target_width = self.hr_upscale_to_x + target_height = self.hr_upscale_to_y + if latent_scale_mode is not None: for i in range(samples.shape[0]): save_intermediate(samples, i) @@ -978,12 +1050,18 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): self.image_conditioning = None def init(self, all_prompts, all_seeds, all_subseeds): + image_mask = self.image_mask + if backend == Backend.DIFFUSERS and image_mask is None: + sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) + elif backend == Backend.DIFFUSERS and image_mask is not None: + sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.INPAINTING) + self.sd_model.dtype = self.sd_model.unet.dtype + force_latent_upscaler = shared.opts.data.get('force_latent_sampler') if self.sampler_name in ['PLMS']: self.sampler_name = force_latent_upscaler if force_latent_upscaler != 'None' else shared.opts.fallback_sampler # PLMS does not support img2img, use fallback instead self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model) crop_region = None - image_mask = self.image_mask if image_mask is not None: image_mask = image_mask.convert('L') if self.inpainting_mask_invert: @@ -1048,7 +1126,13 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): image = torch.from_numpy(batch_images) image = 2. * image - 1. image = image.to(shared.device) - self.init_latent = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(image)) + + if backend == Backend.ORIGINAL: + self.init_latent = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(image)) + else: + # we don't pre-encode the latents for diffusers to allow the UI to stay general for different model types + self.init_latent = None + if self.resize_mode == 3: self.init_latent = torch.nn.functional.interpolate(self.init_latent, size=(self.height // opt_f, self.width // opt_f), mode="bilinear") if image_mask is not None: @@ -1068,6 +1152,13 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): self.image_conditioning = self.img2img_image_conditioning(image, self.init_latent, image_mask) def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): + if backend == Backend.DIFFUSERS: + if self.init_mask is None: # pylint: disable=no-member + sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) + else: + sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.INPAINTING) + self.sd_model.dtype = self.sd_model.unet.dtype + x = create_random_tensors([opt_C, self.height // opt_f, self.width // opt_f], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self) if self.initial_noise_multiplier != 1.0: self.extra_generation_params["Noise multiplier"] = self.initial_noise_multiplier diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 53d899f62..be7e90cc5 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -173,17 +173,17 @@ class StableDiffusionModelHijack: if m.cond_stage_key == "edit": sd_hijack_unet.hijack_ddpm_edit() + import logging if opts.cuda_compile and opts.cuda_compile_mode == 'ipex': - import logging if devices.backend == 'ipex': shared.log.info("Model compile enabled: IPEX Optimize Graph Mode") else: shared.log.warning("Model compile skipped: IPEX Method is for Intel GPU's with OneAPI") - elif opts.cuda_compile and opts.cuda_compile_mode != 'none': + elif opts.cuda_compile and opts.cuda_compile_mode != 'none' and shared.backend == shared.Backend.ORIGINAL: try: - import logging - import torch._dynamo as dynamo # pylint: disable=unused-import - torch._dynamo.config.log_level = logging.WARNING if opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access + import torch._dynamo # pylint: disable=unused-import + log_level = logging.WARNING if opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access + torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access torch._dynamo.config.verbose = opts.cuda_compile_verbose # pylint: disable=protected-access torch._dynamo.config.suppress_errors = opts.cuda_compile_errors # pylint: disable=protected-access torch.backends.cudnn.benchmark = True @@ -191,7 +191,7 @@ class StableDiffusionModelHijack: import hidet hidet.torch.dynamo_config.use_tensor_core(True) hidet.torch.dynamo_config.search_space(2) - m.model = torch.compile(m.model, mode="default", backend=opts.cuda_compile_mode, fullgraph=False, dynamic=False) + m.model = torch.compile(m.model, mode="default", backend=opts.cuda_compile_mode, fullgraph=opts.cuda_compile_fullgraph, dynamic=False) shared.log.info(f"Model compile enabled: {opts.cuda_compile_mode}") except Exception as err: shared.log.warning(f"Model compile not supported: {err}") diff --git a/modules/sd_models.py b/modules/sd_models.py index eb8c5d48f..99a2f2b27 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -6,6 +6,7 @@ import json import threading from os import mkdir from urllib import request +from enum import Enum import filelock from rich import progress # pylint: disable=redefined-builtin import torch @@ -15,12 +16,14 @@ import tomesd from transformers import logging as transformers_logging import ldm.modules.midas as midas from ldm.util import instantiate_from_config +import diffusers from modules import paths, shared, modelloader, devices, script_callbacks, sd_vae, sd_disable_initialization, errors, hashes, sd_models_config from modules.sd_hijack_inpainting import do_inpainting_hijack from modules.timer import Timer from modules.memstats import memory_stats from modules.paths_internal import models_path + transformers_logging.set_verbosity_error() model_dir = "Stable-diffusion" model_path = os.path.abspath(os.path.join(paths.models_path, model_dir)) @@ -39,8 +42,10 @@ class CheckpointInfo: self.name = None self.hash = None self.filename = filename + self.type = '' abspath = os.path.abspath(filename) - if shared.backend == shared.Backend.ORIGINAL: + + if os.path.isfile(abspath): # ckpt or safetensor if shared.opts.ckpt_dir is not None and abspath.startswith(shared.opts.ckpt_dir): name = abspath.replace(shared.opts.ckpt_dir, '') elif abspath.startswith(model_path): @@ -52,7 +57,11 @@ class CheckpointInfo: self.name = name self.hash = model_hash(self.filename) self.sha256 = hashes.sha256_from_cache(self.filename, f"checkpoint/{name}") - else: # TODO Diffusers + self.path = abspath + self.type = abspath.split('.')[-1].lower() + self.name_for_extra = os.path.splitext(os.path.basename(filename))[0] + self.model_name = os.path.splitext(name.replace("/", "_").replace("\\", "_"))[0] + else: # maybe a diffuser repo = [r for r in modelloader.diffuser_repos if filename == r['filename']] if len(repo) == 0: error_message = f'Cannot find diffuser model: {filename}' @@ -61,8 +70,15 @@ class CheckpointInfo: self.name = repo[0]['name'] self.hash = repo[0]['hash'][:8] self.sha256 = repo[0]['hash'] - self.name_for_extra = os.path.splitext(os.path.basename(filename))[0] - self.model_name = os.path.splitext(name.replace("/", "_").replace("\\", "_"))[0] + self.path = repo[0]['path'] + self.type = 'diffusers' + self.name_for_extra = repo[0]['name'] + self.model_name = repo[0]['name'] + if os.path.isfile(repo[0]['model_info']): + file_path = repo[0]['model_info'] + with open(file_path, "r", encoding="utf-8") as json_file: + self.model_info = json.load(json_file) + self.shorthash = self.sha256[0:10] if self.sha256 else None self.title = self.name if self.shorthash is None else f'{self.name} [{self.shorthash}]' self.ids = [self.hash, self.model_name, self.title, self.name, f'{self.name} [{self.hash}]'] + ([self.shorthash, self.sha256, f'{self.name} [{self.shorthash}]'] if self.shorthash else []) @@ -110,13 +126,13 @@ def checkpoint_tiles(): def list_models(): checkpoints_list.clear() checkpoint_aliases.clear() - if shared.backend == shared.Backend.ORIGINAL: - ext_filter=[".safetensors"] if shared.opts.sd_disable_ckpt else [".ckpt", ".safetensors"] - model_list = modelloader.load_models(model_path=os.path.join(models_path, 'Stable-diffusion'), model_url=None, command_path=shared.opts.ckpt_dir, ext_filter=ext_filter, download_name=None, ext_blacklist=[".vae.ckpt", ".vae.safetensors"]) - else: - global model_path # pylint: disable=global-statement - model_path = os.path.join(models_path, 'Diffusers') - model_list = modelloader.load_diffusers(model_path=model_path, command_path=shared.opts.diffusers_dir) + ext_filter=[".safetensors"] if shared.opts.sd_disable_ckpt else [".ckpt", ".safetensors"] + model_list = [] + if shared.backend == shared.Backend.ORIGINAL or shared.opts.diffusers_allow_safetensors: + model_list += modelloader.load_models(model_path=model_path, model_url=None, command_path=shared.opts.ckpt_dir, ext_filter=ext_filter, download_name=None, ext_blacklist=[".vae.ckpt", ".vae.safetensors"]) + if shared.backend == shared.Backend.DIFFUSERS: + model_list += modelloader.load_diffusers_models(model_path=os.path.join(models_path, 'Diffusers'), command_path=shared.opts.diffusers_dir) + for filename in sorted(model_list, key=str.lower): checkpoint_info = CheckpointInfo(filename) if checkpoint_info.name is not None: @@ -143,15 +159,15 @@ def list_models(): shared.opts.data['sd_model_checkpoint'] = "v1-5-pruned-emaonly.safetensors" model_list = modelloader.load_models(model_path=model_path, model_url=model_url, command_path=shared.opts.ckpt_dir, ext_filter=[".ckpt", ".safetensors"], download_name="v1-5-pruned-emaonly.safetensors", ext_blacklist=[".vae.ckpt", ".vae.safetensors"]) else: - hub_url = "runwayml/stable-diffusion-v1-5" - model_list = modelloader.load_diffusers(model_path=model_path, hub_url=hub_url, command_path=shared.opts.diffusers_dir) + default_model_id = "runwayml/stable-diffusion-v1-5" + modelloader.download_diffusers_model(default_model_id, shared.opts.diffusers_dir) + model_list = modelloader.load_diffusers_models(model_path=os.path.join(models_path, 'Diffusers'), command_path=shared.opts.diffusers_dir) for filename in sorted(model_list, key=str.lower): checkpoint_info = CheckpointInfo(filename) if checkpoint_info.name is not None: checkpoint_info.register() - def update_model_hashes(): txt = [] lst = [ckpt for ckpt in checkpoints_list.values() if ckpt.hash is None] @@ -196,11 +212,18 @@ def model_hash(filename): return 'NOHASH' -def select_checkpoint(model=True): - model_checkpoint = shared.opts.sd_model_checkpoint if model else shared.opts.sd_model_dict +def select_checkpoint(op='model'): + if op == 'model': + model_checkpoint = shared.opts.sd_model_checkpoint + elif op == 'dict': + model_checkpoint = shared.opts.sd_model_dict + elif op == 'refiner': + model_checkpoint = shared.opts.data.get('sd_model_refiner', None) + if model_checkpoint is None or model_checkpoint == 'None': + return None checkpoint_info = get_closet_checkpoint_match(model_checkpoint) if checkpoint_info is not None: - shared.log.debug(f'Select checkpoint: {checkpoint_info.title if checkpoint_info is not None else None}') + shared.log.debug(f'Select checkpoint: {op} {checkpoint_info.title if checkpoint_info is not None else None}') return checkpoint_info if len(checkpoints_list) == 0: shared.log.error("Cannot run without a checkpoint") @@ -209,7 +232,8 @@ def select_checkpoint(model=True): checkpoint_info = next(iter(checkpoints_list.values())) if model_checkpoint is not None: shared.log.warning(f"Selected checkpoint not found: {model_checkpoint}") - shared.log.warning(f"Loading fallback checkpoint: {checkpoint_info.title}") + # shared.log.warning(f"Loading fallback checkpoint: {checkpoint_info.title}") + shared.opts.data['sd_checkpoint'] = checkpoint_info.title shared.log.debug(f'Select checkpoint: {checkpoint_info.title if checkpoint_info is not None else None}') return checkpoint_info @@ -443,9 +467,10 @@ sd1_clip_weight = 'cond_stage_model.transformer.text_model.embeddings.token_embe sd2_clip_weight = 'cond_stage_model.model.transformer.resblocks.0.attn.in_proj_weight' -class SdModelData: +class ModelData: def __init__(self): self.sd_model = None + self.sd_refiner = None self.sd_dict = 'None' self.initial = True self.lock = threading.Lock() @@ -455,11 +480,11 @@ class SdModelData: with self.lock: try: if shared.backend == shared.Backend.ORIGINAL: - reload_model_weights() + reload_model_weights(op='model') elif shared.backend == shared.Backend.DIFFUSERS: - load_diffuser() + load_diffuser(op='model') else: - shared.log.error(f"Unknown Stable Diffusion backend: {shared.opts.sd_backend}") + shared.log.error(f"Unknown Stable Diffusion backend: {shared.backend}") self.initial = False except Exception as e: shared.log.error("Failed to load stable diffusion model") @@ -468,82 +493,346 @@ class SdModelData: return self.sd_model def set_sd_model(self, v): + shared.log.debug(f"Class model: {v}") self.sd_model = v + def get_sd_refiner(self): + if self.sd_model is None: + with self.lock: + try: + if shared.backend == shared.Backend.ORIGINAL: + reload_model_weights(op='refiner') + elif shared.backend == shared.Backend.DIFFUSERS: + load_diffuser(op='refiner') + else: + shared.log.error(f"Unknown Stable Diffusion backend: {shared.backend}") + self.initial = False + except Exception as e: + shared.log.error("Failed to load stable diffusion model") + errors.display(e, "loading stable diffusion model") + self.sd_refiner = None + return self.sd_refiner -model_data = SdModelData() + def set_sd_refiner(self, v): + shared.log.debug(f"Class refiner: {v}") + self.sd_refiner = v + +model_data = ModelData() + +class PriorPipeline: + def __init__(self, prior, main): + self.prior = prior + self.main = main + self.scheduler = main.scheduler + self.tokenizer = self.prior.tokenizer + + def to(self, *args, **kwargs): + # only the prior is moved to CUDA in a first step + self.prior.to(*args, **kwargs) + + def enable_model_cpu_offload(self, *args, **kwargs): + self.prior.enable_model_cpu_offload(*args, **kwargs) + self.main.enable_model_cpu_offload(*args, **kwargs) + + def enable_sequential_cpu_offload(self, *args, **kwargs): + self.prior.enable_sequential_cpu_offload(*args, **kwargs) + self.main.enable_sequential_cpu_offload(*args, **kwargs) + + def enable_xformers_memory_efficient_attention(self, *args, **kwargs): + self.prior.enable_xformers_memory_efficient_attention(*args, **kwargs) + self.main.enable_xformers_memory_efficient_attention(*args, **kwargs) + + def __call__(self, *args, **kwargs): + unclip_outputs = self.prior(prompt=kwargs.get("prompt"), negative_prompt=kwargs.get("negative_prompt")) + + if self.prior.device.type == "cuda" or self.prior.device.type == "xpu": + prior_device = self.prior.device + self.prior.to("cpu") + self.main.to(prior_device) + + kwargs = {**kwargs, **unclip_outputs} + result = self.main(*args, **kwargs) + + if self.main.device.type == "cuda" or self.main.device.type == "xpu": + main_device = self.main.device + self.main.to("cpu") + self.prior.to(main_device) + + return result -def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None): # pylint: disable=unused-argument +def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument + import torch # pylint: disable=reimported,redefined-outer-name if timer is None: timer = Timer() - import diffusers import logging logging.getLogger("diffusers").setLevel(logging.ERROR) timer.record("diffusers") - diffusor_config = { - "force_download": False, - "safety_checker": None, - "resume_download": True, + diffusers_load_config = { "low_cpu_mem_usage": True, - "use_safetensors": True, - "cache_dir": shared.opts.diffusers_dir, "torch_dtype": devices.dtype, + "safety_checker": None, + "requires_safety_checker": False, + "load_safety_checker": False, + # "use_safetensors": True, # TODO(PVP) - we can't enable this for all checkpoints just yet } - if shared.opts.data['sd_model_checkpoint'] == 'model.ckpt': + if devices.dtype == torch.float16: + diffusers_load_config['variant'] = 'fp16' + + if shared.opts.data.get('sd_model_checkpoint', '') == 'model.ckpt' or shared.opts.data.get('sd_model_checkpoint', '') == '': shared.opts.data['sd_model_checkpoint'] = "runwayml/stable-diffusion-v1-5" + + if op == 'model' or op == 'dict': + if (model_data.sd_model is not None) and (checkpoint_info is not None) and (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model + return + else: + if (model_data.sd_refiner is not None) and (checkpoint_info is not None) and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model + return + + shared.log.debug(f'Diffusers load config: {diffusers_load_config}') sd_model = None try: + devices.set_cuda_params() if shared.cmd_opts.ckpt is not None and model_data.initial: # initial load model_name = modelloader.find_diffuser(shared.cmd_opts.ckpt) if model_name is not None: - shared.log.info(f'Loading diffuser model: {model_name}') - scheduler = diffusers.UniPCMultistepScheduler.from_pretrained(model_name, subfolder="scheduler") - sd_model = diffusers.DiffusionPipeline.from_pretrained(model_name, scheduler=scheduler, **diffusor_config) + shared.log.info(f'Loading diffuser {op}: {model_name}') + model_file = modelloader.download_diffusers_model(hub_id=model_name) + try: + sd_model = diffusers.DiffusionPipeline.from_pretrained(model_file, **diffusers_load_config) + except Exception as e: + shared.log.error(f'Diffusers failed loading model: {model_file} {e}') list_models() # rescan for downloaded model checkpoint_info = CheckpointInfo(model_name) + if sd_model is None: - checkpoint_info = checkpoint_info or select_checkpoint() - shared.log.info(f'Loading diffuser model: {checkpoint_info.filename}') - scheduler = diffusers.UniPCMultistepScheduler.from_pretrained(checkpoint_info.filename, subfolder="scheduler") - sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.filename, scheduler=scheduler, **diffusor_config) - if shared.cmd_opts.medvram: - sd_model.enable_model_cpu_offload() - if shared.cmd_opts.lowvram: - sd_model.enable_sequential_cpu_offload() + checkpoint_info = checkpoint_info or select_checkpoint(op=op) + if checkpoint_info is None: + unload_model_weights(op=op) + return + shared.log.info(f'Loading diffuser {op}: {checkpoint_info.filename}') + + vae_file, vae_source = sd_vae.resolve_vae(checkpoint_info.filename) + vae = sd_vae.load_vae_diffusers(None, vae_file, vae_source) + if vae is not None: + diffusers_load_config["vae"] = vae + + if not os.path.isfile(checkpoint_info.path): + try: + sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) + except Exception as e: + shared.log.error(f'Diffusers failed loading model: {checkpoint_info.path} {e}') + else: + diffusers_load_config["local_files_only "] = True + diffusers_load_config["extract_ema"] = shared.opts.diffusers_extract_ema + try: + if shared.opts.diffusers_pipeline == shared.pipelines[0]: + pipeline = diffusers.StableDiffusionPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[1]: + pipeline = diffusers.StableDiffusionXLPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[2]: + pipeline = diffusers.KandinskyPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[3]: + pipeline = diffusers.KandinskyV22Pipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[4]: + pipeline = diffusers.IFPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[5]: + pipeline = diffusers.ShapEPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[6]: + pipeline = diffusers.StableDiffusionImg2ImgPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[7]: + pipeline = diffusers.StableDiffusionXLImg2ImgPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[8]: + pipeline = diffusers.KandinskyImg2ImgPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[9]: + pipeline = diffusers.KandinskyV22Img2ImgPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[10]: + pipeline = diffusers.IFImg2ImgPipeline + elif shared.opts.diffusers_pipeline == shared.pipelines[11]: + pipeline = diffusers.ShapEImg2ImgPipeline + else: + shared.log.error(f'Diffusers unknown pipeline: {shared.opts.diffusers_pipeline}') + except Exception as e: + shared.log.error(f'Diffusers failed initializing pipeline: {shared.opts.diffusers_pipeline} {e}') + return + try: + if hasattr(pipeline, 'from_single_file'): + diffusers_load_config['use_safetensors'] = True + sd_model = pipeline.from_single_file(checkpoint_info.path, **diffusers_load_config) + elif hasattr(pipeline, 'from_ckpt'): + sd_model = pipeline.from_ckpt(checkpoint_info.path, **diffusers_load_config) + else: + shared.log.error(f'Diffusers cannot load safetensor model: {checkpoint_info.path} {shared.opts.diffusers_pipeline}') + return + if sd_model is not None: + shared.log.debug(f'Diffusers pipeline: {type(sd_model)}') # pylint: disable=protected-access + except Exception as e: + shared.log.error(f'Diffusers failed loading model using pipeline: {checkpoint_info.path} {shared.opts.diffusers_pipeline} {e}') + return + + if "StableDiffusion" in sd_model.__class__.__name__: + pass # scheduler is created on first use + elif "Kandinsky" in sd_model.__class__.__name__: + sd_model.scheduler.name = 'DDIM' + + # Prior pipelines + if hasattr(checkpoint_info, 'model_info') and checkpoint_info.model_info is not None and "prior" in checkpoint_info.model_info: + prior_id = checkpoint_info.model_info["prior"] + shared.log.info(f"Loading diffuser prior: {checkpoint_info.filename} {prior_id}") + prior = diffusers.DiffusionPipeline.from_pretrained(prior_id, **diffusers_load_config) + sd_model = PriorPipeline(prior=prior, main=sd_model) # wrap sd_model + + if hasattr(sd_model, "enable_model_cpu_offload"): + if shared.cmd_opts.medvram or shared.opts.diffusers_model_cpu_offload: + shared.log.debug('Diffusers: enable model CPU offload') + sd_model.enable_model_cpu_offload() + if hasattr(sd_model, "enable_sequential_cpu_offload"): + if shared.opts.diffusers_seq_cpu_offload: + sd_model.enable_sequential_cpu_offload() + shared.log.debug('Diffusers: enable sequential CPU offload') + if hasattr(sd_model, "enable_vae_slicing"): + if shared.cmd_opts.lowvram or shared.opts.diffusers_vae_slicing: + shared.log.debug('Diffusers: enable VAE slicing') + sd_model.enable_vae_slicing() + else: + sd_model.disable_vae_slicing() + if hasattr(sd_model, "enable_vae_tiling"): + if shared.cmd_opts.lowvram or shared.opts.diffusers_vae_tiling: + shared.log.debug('Diffusers: enable VAE tiling') + sd_model.enable_vae_tiling() + else: + sd_model.disable_vae_tiling() + if hasattr(sd_model, "enable_attention_slicing"): + if shared.cmd_opts.lowvram or shared.opts.diffusers_attention_slicing: + shared.log.debug('Diffusers: enable attention slicing') + sd_model.enable_attention_slicing() + else: + sd_model.disable_attention_slicing() if shared.opts.cross_attention_optimization == "xFormers": sd_model.enable_xformers_memory_efficient_attention() - sd_model.sd_checkpoint_info = checkpoint_info - sd_model.sd_model_checkpoint = checkpoint_info.filename - sd_model.sd_model_hash = checkpoint_info.hash - scheduler.name = 'UniPC' + if shared.opts.opt_channelslast: + shared.log.debug('Diffusers: enable channels last') + sd_model.unet.to(memory_format=torch.channels_last) + if shared.opts.cuda_compile and torch.cuda.is_available(): + sd_model.to(devices.device) + import torch._dynamo # pylint: disable=unused-import,redefined-outer-name + log_level = logging.WARNING if shared.opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access + torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access + torch._dynamo.config.verbose = shared.opts.cuda_compile_verbose # pylint: disable=protected-access + torch._dynamo.config.suppress_errors = shared.opts.cuda_compile_errors # pylint: disable=protected-access + sd_model.unet = torch.compile(sd_model.unet, mode=shared.opts.cuda_compile_mode, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init + shared.log.info(f"Compiling pipeline={sd_model.__class__.__name__} shape={8 * sd_model.unet.config.sample_size} mode={shared.opts.cuda_compile_mode}") + sd_model("dummy prompt") + shared.log.info("Complilation done.") + + if sd_model is None: + shared.log.error('Diffuser model not loaded') + return + sd_model.sd_checkpoint_info = checkpoint_info # pylint: disable=attribute-defined-outside-init + sd_model.sd_model_checkpoint = checkpoint_info.filename # pylint: disable=attribute-defined-outside-init + sd_model.sd_model_hash = checkpoint_info.hash # pylint: disable=attribute-defined-outside-init sd_model.to(devices.device) except Exception as e: shared.log.error("Failed to load diffusers model") errors.display(e, "loading Diffusers model") - shared.sd_model = sd_model + + if op == 'refiner': + model_data.sd_refiner = sd_model + else: + model_data.sd_model = sd_model + + from modules.textual_inversion import textual_inversion + embedding_db = textual_inversion.EmbeddingDatabase() + embedding_db.add_embedding_dir(shared.opts.embeddings_dir) + embedding_db.load_textual_inversion_embeddings(force_reload=True) + timer.record("load") shared.log.info(f"Model loaded in {timer.summary()}") devices.torch_gc(force=True) shared.log.info(f'Model load finished: {memory_stats()}') +class DiffusersTaskType(Enum): + TEXT_2_IMAGE = 1 + IMAGE_2_IMAGE = 2 + INPAINTING = 3 -def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None): +def set_diffuser_pipe(pipe, new_pipe_type): + wrapper_pipe = None + + sd_checkpoint_info = pipe.sd_checkpoint_info + sd_model_checkpoint = pipe.sd_model_checkpoint + sd_model_hash = pipe.sd_model_hash + + if pipe.__class__ == PriorPipeline: + wrapper_pipe = pipe + pipe = pipe.main + + pipe_name = pipe.__class__.__name__ + pipe_name = pipe_name.replace("Img2Img", "").replace("Inpaint", "") + if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: + new_pipe_cls_str = pipe_name + elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: + new_pipe_cls_str = pipe_name.replace("Pipeline", "Img2ImgPipeline") + elif new_pipe_type == DiffusersTaskType.INPAINTING: + new_pipe_cls_str = pipe_name.replace("Pipeline", "InpaintPipeline") + + new_pipe_cls = getattr(diffusers, new_pipe_cls_str) + + if pipe.__class__ == new_pipe_cls: + return + + new_pipe = new_pipe_cls(**pipe.components) + + if wrapper_pipe is not None: + wrapper_pipe.main = new_pipe + new_pipe = wrapper_pipe + + new_pipe.sd_checkpoint_info = sd_checkpoint_info + new_pipe.sd_model_checkpoint = sd_model_checkpoint + new_pipe.sd_model_hash = sd_model_hash + + model_data.sd_model = new_pipe + shared.log.info(f"Pipeline class changed from {pipe.__class__.__name__} to {new_pipe_cls.__name__}") + + +def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType: + if pipe.__class__ == PriorPipeline: + pipe = pipe.main + + if "Img2Img" in pipe.__class__.__name__: + return DiffusersTaskType.IMAGE_2_IMAGE + elif "Inpaint" in pipe.__class__.__name__: + return DiffusersTaskType.INPAINTING + return DiffusersTaskType.TEXT_2_IMAGE + + +def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): from modules import lowvram, sd_hijack - checkpoint_info = checkpoint_info or select_checkpoint() + checkpoint_info = checkpoint_info or select_checkpoint(op=op) if checkpoint_info is None: return - if model_data.sd_model is not None and (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model - return - shared.log.debug(f'Load model: name={checkpoint_info.filename} dict={already_loaded_state_dict is not None}') + if op == 'model' or op == 'dict': + if model_data.sd_model is not None and (checkpoint_info.hash == model_data.sd_model.sd_checkpoint_info.hash): # trying to load the same model + return + else: + if model_data.sd_refiner is not None and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model + return + shared.log.debug(f'Load {op}: name={checkpoint_info.filename} dict={already_loaded_state_dict is not None}') if timer is None: timer = Timer() current_checkpoint_info = None - if model_data.sd_model is not None: - sd_hijack.model_hijack.undo_hijack(model_data.sd_model) - current_checkpoint_info = model_data.sd_model.sd_checkpoint_info - unload_model_weights() + if op == 'model' or op == 'dict': + if model_data.sd_model is not None: + sd_hijack.model_hijack.undo_hijack(model_data.sd_model) + current_checkpoint_info = model_data.sd_model.sd_checkpoint_info + unload_model_weights(op=op) + else: + if model_data.sd_refiner is not None: + sd_hijack.model_hijack.undo_hijack(model_data.sd_refiner) + current_checkpoint_info = model_data.sd_refiner.sd_checkpoint_info + unload_model_weights(op=op) + do_inpainting_hijack() devices.set_cuda_params() if already_loaded_state_dict is not None: @@ -589,7 +878,10 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None) sd_model = torch.xpu.optimize(sd_model, dtype=devices.dtype, auto_kernel_selection=True, optimize_lstm=True, graph_mode=True if shared.opts.cuda_compile and shared.opts.cuda_compile_mode == 'ipex' else False) shared.log.info("Applied IPEX Optimize") - model_data.sd_model = sd_model + if op == 'refiner': + model_data.sd_refiner = sd_model + else: + model_data.sd_model = sd_model sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings(force_reload=True) # Reload embeddings after model load as they may or may not fit the model timer.record("embeddings") script_callbacks.model_loaded_callback(sd_model) @@ -600,7 +892,7 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None) shared.log.info(f'Model load finished: {memory_stats()} cached={len(checkpoints_loaded.keys())}') -def reload_model_weights(sd_model=None, info=None, reuse_dict=False): +def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model'): load_dict = shared.opts.sd_model_dict != model_data.sd_dict global skip_next_load # pylint: disable=global-statement if skip_next_load: @@ -608,15 +900,18 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False): skip_next_load = False return from modules import lowvram, sd_hijack - checkpoint_info = info or select_checkpoint(model=not load_dict) # are we selecting model or dictionary - next_checkpoint_info = info or select_checkpoint(model=load_dict) if load_dict else None + checkpoint_info = info or select_checkpoint(op=op) # are we selecting model or dictionary + next_checkpoint_info = info or select_checkpoint(op='dict' if load_dict else 'model') if load_dict else None + if checkpoint_info is None: + unload_model_weights(op=op) + return if load_dict: shared.log.debug(f'Model dict: existing={sd_model is not None} target={checkpoint_info.filename} info={info}') else: model_data.sd_dict = 'None' shared.log.debug(f'Load model weights: existing={sd_model is not None} target={checkpoint_info.filename} info={info}') if not sd_model: - sd_model = model_data.sd_model + sd_model = model_data.sd_model if op == 'model' or op == 'dict' else model_data.sd_refiner if sd_model is None: # previous model load failed current_checkpoint_info = None else: @@ -631,7 +926,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False): shared.log.info('Reusing previous model dictionary') sd_hijack.model_hijack.undo_hijack(sd_model) else: - unload_model_weights() + unload_model_weights(op=op) sd_model = None timer = Timer() state_dict = get_checkpoint_state_dict(checkpoint_info, timer) @@ -640,14 +935,14 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False): if sd_model is None or checkpoint_config != sd_model.used_config: del sd_model if shared.backend == shared.Backend.ORIGINAL: - load_model(checkpoint_info, already_loaded_state_dict=state_dict, timer=timer) + load_model(checkpoint_info, already_loaded_state_dict=state_dict, timer=timer, op=op) else: - load_diffuser(checkpoint_info, already_loaded_state_dict=state_dict, timer=timer) + load_diffuser(checkpoint_info, already_loaded_state_dict=state_dict, timer=timer, op=op) if load_dict and next_checkpoint_info is not None: model_data.sd_dict = shared.opts.sd_model_dict shared.opts.data["sd_model_checkpoint"] = next_checkpoint_info.title reload_model_weights(reuse_dict=True) # ok we loaded dict now lets redo and load model on top of it - return model_data.sd_model + return model_data.sd_model if op == 'model' or op == 'dict' else model_data.sd_refiner try: load_model_weights(sd_model, checkpoint_info, state_dict, timer) except Exception: @@ -664,17 +959,23 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False): shared.log.info(f"Weights loaded in {timer.summary()}") -def unload_model_weights(sd_model=None, _info=None): +def unload_model_weights(op='model'): from modules import sd_hijack - if model_data.sd_model: - model_data.sd_model.to(devices.cpu) - if shared.backend == shared.Backend.ORIGINAL: - sd_hijack.model_hijack.undo_hijack(model_data.sd_model) - sd_model = None - model_data.sd_model = None - devices.torch_gc(force=True) - shared.log.debug(f'Model weights unloaded: {memory_stats()}') - return sd_model + if op == 'model' or op == 'dict': + if model_data.sd_model: + model_data.sd_model.to(devices.cpu) + if shared.backend == shared.Backend.ORIGINAL: + sd_hijack.model_hijack.undo_hijack(model_data.sd_model) + model_data.sd_model = None + shared.log.debug(f'Weights unloaded {op}: {memory_stats()}') + else: + if model_data.sd_refiner: + model_data.sd_refiner.to(devices.cpu) + if shared.backend == shared.Backend.ORIGINAL: + sd_hijack.model_hijack.undo_hijack(model_data.sd_refiner) + model_data.sd_refiner = None + shared.log.debug(f'Weights unloaded {op}: {memory_stats()}') + devices.torch_gc(force=True) def apply_token_merging(sd_model, token_merging_ratio): @@ -684,6 +985,11 @@ def apply_token_merging(sd_model, token_merging_ratio): return if current_token_merging_ratio > 0: tomesd.remove_patch(sd_model) + + if sd_model.__class__ == PriorPipeline: + # token merging is not supported for PriorPipelines currently + return + if token_merging_ratio > 0: tomesd.apply_patch( sd_model, diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index eeda69d0b..140286064 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -1,16 +1,11 @@ -from modules import sd_samplers_compvis, sd_samplers_kdiffusion, sd_samplers_diffusors, shared +from modules import sd_samplers_compvis, sd_samplers_kdiffusion, sd_samplers_diffusers, shared from modules.sd_samplers_common import samples_to_image_grid, sample_to_image # pylint: disable=unused-import from modules.shared import backend, Backend if backend == Backend.ORIGINAL: - all_samplers = [ - *sd_samplers_kdiffusion.samplers_data_k_diffusion, - *sd_samplers_compvis.samplers_data_compvis, - ] + all_samplers = [*sd_samplers_kdiffusion.samplers_data_k_diffusion, *sd_samplers_compvis.samplers_data_compvis] else: - all_samplers = [ - *sd_samplers_diffusors.samplers_data_diffusors, - ] + all_samplers = [*sd_samplers_diffusers.samplers_data_diffusers] all_samplers_map = {x.name: x for x in all_samplers} samplers = all_samplers samplers_for_img2img = all_samplers @@ -34,10 +29,12 @@ def create_sampler(name, model): sampler = config.constructor(model) sampler.config = config return sampler - else: - sampler = config.constructor(model.sd_checkpoint_info.filename) + elif backend == Backend.DIFFUSERS: + sampler = config.constructor(model) model.scheduler = sampler.sampler return sampler.sampler + else: + return None def set_samplers(): diff --git a/modules/sd_samplers_compvis.py b/modules/sd_samplers_compvis.py index 96bf4d129..562705e1c 100644 --- a/modules/sd_samplers_compvis.py +++ b/modules/sd_samplers_compvis.py @@ -37,7 +37,6 @@ class VanillaStableDiffusionSampler: self.eta = None self.config = None self.last_latent = None - self.conditioning_key = sd_model.model.conditioning_key def number_of_needed_noises(self, p): # pylint: disable=unused-argument diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py new file mode 100644 index 000000000..fc32513da --- /dev/null +++ b/modules/sd_samplers_diffusers.py @@ -0,0 +1,71 @@ +from diffusers import ( + DDIMScheduler, + DDPMScheduler, + DEISMultistepScheduler, + DPMSolverMultistepScheduler, + DPMSolverSinglestepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + HeunDiscreteScheduler, + # KDPM2DiscreteScheduler, + PNDMScheduler, + UniPCMultistepScheduler, +) +from modules import sd_samplers_common + +config = { + 'All': { 'num_train_timesteps': 1000, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'linear', 'prediction_type': 'epsilon' }, + 'UniPC': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True }, + 'DDIM': { 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'thresholding': False, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'leading', 'rescale_betas_zero_snr': False }, + 'DDPM': { 'variance_type': "fixed_small", 'clip_sample': True, 'thresholding': False, 'clip_sample_range': 1.0, 'sample_max_value': 1.0 }, + 'DEIS': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "deis", 'solver_type': "logrho", 'lower_order_final': True }, + 'Euler': { 'interpolation_type': "linear", 'use_karras_sigmas': False }, + 'Euler a': {}, + 'Heun': { 'use_karras_sigmas': False }, + 'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0 }, + 'DPM 1S': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False }, + 'DPM 2M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False }, +} + +samplers_data_diffusers = [ + sd_samplers_common.SamplerData('UniPC', lambda model: DiffusionSampler('UniPC', UniPCMultistepScheduler, model), [], {}), + sd_samplers_common.SamplerData('DDIM', lambda model: DiffusionSampler('DDIM', DDIMScheduler, model), [], {}), + sd_samplers_common.SamplerData('DDPM', lambda model: DiffusionSampler('DDPM', DDPMScheduler, model), [], {}), + sd_samplers_common.SamplerData('DEIS', lambda model: DiffusionSampler('DEIS', DEISMultistepScheduler, model), [], {}), + sd_samplers_common.SamplerData('DPM 1S', lambda model: DiffusionSampler('DPM++ 1S', DPMSolverSinglestepScheduler, model), [], {}), + sd_samplers_common.SamplerData('DPM 2M', lambda model: DiffusionSampler('DPM++ 2M', DPMSolverMultistepScheduler, model), [], {}), + sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), + sd_samplers_common.SamplerData('Euler a', lambda model: DiffusionSampler('Euler a', EulerAncestralDiscreteScheduler, model), [], {}), + sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), + sd_samplers_common.SamplerData('PNDM', lambda model: DiffusionSampler('PNDM', PNDMScheduler, model), [], {}), +] + +class DiffusionSampler: + def __init__(self, name, constructor, model, **kwargs): + from modules.shared import opts, log + self.config = config['All'].copy() + for key, value in config.get(name, {}).items(): # diffusers defaults + self.config[key] = value + for key, value in model.scheduler.config.items(): # model defaults + if key in self.config: + self.config[key] = value + for key, value in kwargs.items(): # user args + if key in self.config: + self.config[key] = value + if opts.schedulers_prediction_type != 'default': + self.config['prediction_type'] = opts.schedulers_prediction_type + if opts.schedulers_beta_schedule != 'default': + self.config['beta_schedule'] = opts.schedulers_beta_schedule + if 'use_karras_sigmas' in self.config: + self.config['use_karras_sigmas'] = opts.schedulers_use_karras + if 'thresholding' in self.config: + self.config['thresholding'] = opts.schedulers_use_thresholding + if 'lower_order_final' in self.config: + self.config['lower_order_final'] = opts.schedulers_use_loworder + if 'solver_order' in self.config: + self.config['solver_order'] = opts.schedulers_solver_order + if name.startswith('DPM'): + self.config['algorithm_type'] = opts.schedulers_dpm_solver + self.sampler = constructor(**self.config) + self.sampler.name = name + log.debug(f'Diffusers sampler: {name} {self.config}') diff --git a/modules/sd_samplers_diffusors.py b/modules/sd_samplers_diffusors.py deleted file mode 100644 index 3d102dca4..000000000 --- a/modules/sd_samplers_diffusors.py +++ /dev/null @@ -1,45 +0,0 @@ -from diffusers import ( - DDIMScheduler, - DDPMScheduler, - DEISMultistepScheduler, - DPMSolverMultistepScheduler, - EulerAncestralDiscreteScheduler, - EulerDiscreteScheduler, - HeunDiscreteScheduler, - IPNDMScheduler, - KDPM2AncestralDiscreteScheduler, - PNDMScheduler, - UniPCMultistepScheduler, - # KarrasVeScheduler, - # RePaintScheduler, - # ScoreSdeVeScheduler, - # UnCLIPScheduler, - # VQDiffusionScheduler, -) -from modules import sd_samplers_common - # scheduler = diffusers.UniPCMultistepScheduler.from_pretrained(shared.cmd_opts.ckpt, subfolder="scheduler") - -samplers_data_diffusors = [ - sd_samplers_common.SamplerData('UniPC', lambda model: DiffusionSampler('UniPC', UniPCMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DDIM', lambda model: DiffusionSampler('DDIM', DDIMScheduler, model), [], {}), - sd_samplers_common.SamplerData('DDPMS', lambda model: DiffusionSampler('DDPMS', DDPMScheduler, model), [], {}), - sd_samplers_common.SamplerData('DEIS', lambda model: DiffusionSampler('DEIS', DEISMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPMSolver', lambda model: DiffusionSampler('DPMSolver', DPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('EulerAncestral', lambda model: DiffusionSampler('EulerAncestral', EulerAncestralDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('IPNDM', lambda model: DiffusionSampler('IPNDM', IPNDMScheduler, model), [], {}), - sd_samplers_common.SamplerData('KDPM2Ancestral', lambda model: DiffusionSampler('KDPM2Ancestral', KDPM2AncestralDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('PNDMS', lambda model: DiffusionSampler('PNDMS', PNDMScheduler, model), [], {}), - # sd_samplers_common.SamplerData('KarrasVe', lambda model: DiffusionSampler('KarrasVe', KarrasVeScheduler, model), [], {}), - # sd_samplers_common.SamplerData('RePaint', lambda model: DiffusionSampler('RePaint', RePaintScheduler, model), [], {}), - # sd_samplers_common.SamplerData('ScoreSdeVe', lambda model: DiffusionSampler('ScoreSdeVe', ScoreSdeVeScheduler, model), [], {}), - # sd_samplers_common.SamplerData('UnCLIP', lambda model: DiffusionSampler('UnCLIP', UnCLIPScheduler, model), [], {}), - # sd_samplers_common.SamplerData('VQDiffusion', lambda model: DiffusionSampler('VQDiffusion', VQDiffusionScheduler, model), [], {}), -] - - -class DiffusionSampler: - def __init__(self, name, constructor, sd_model): - self.sampler = constructor.from_pretrained(sd_model, subfolder="scheduler") - self.sampler.name = name diff --git a/modules/sd_vae.py b/modules/sd_vae.py index 1943ec5f9..38ffde339 100644 --- a/modules/sd_vae.py +++ b/modules/sd_vae.py @@ -5,6 +5,7 @@ from copy import deepcopy import torch from modules import shared, paths, devices, script_callbacks, sd_models + vae_ignore_keys = {"model_ema.decay", "model_ema.num_updates"} vae_dict = {} base_vae = None @@ -13,6 +14,7 @@ checkpoint_info = None vae_path = os.path.abspath(os.path.join(paths.models_path, 'VAE')) checkpoints_loaded = collections.OrderedDict() + def get_base_vae(model): if base_vae is not None and checkpoint_info == model.sd_checkpoint_info and model: return base_vae @@ -147,6 +149,34 @@ def load_vae(model, vae_file=None, vae_source="from unknown source"): loaded_vae_file = vae_file +def load_vae_diffusers(_model, vae_file=None, vae_source="from unknown source"): + global loaded_vae_file # pylint: disable=global-statement + if loaded_vae_file == vae_file: + return + loaded_vae_file = None + if vae_file is None: + return + if not os.path.isfile(vae_file): + shared.log.error('VAE not found: {vae_file}') + return + shared.log.info(f"Loading diffusers VAE: {vae_source}: {vae_file}") + diffusers_load_config = { + "low_cpu_mem_usage": True, + "torch_dtype": devices.dtype_vae, + "use_safetensors": True, + } + if devices.dtype_vae == torch.float16: + diffusers_load_config['variant'] = 'fp16' + shared.log.debug(f'Diffusers VAE load config: {diffusers_load_config}') + try: + import diffusers + diffusers_vae = diffusers.AutoencoderKL.from_pretrained(vae_file, **diffusers_load_config) + except Exception as e: + shared.log.error(f"Loading diffusers VAE failed: {vae_file} {e}") + diffusers_vae = None + return diffusers_vae + + # don't call this from outside def _load_vae_dict(model, vae_dict_1): model.first_stage_model.load_state_dict(vae_dict_1) @@ -178,12 +208,17 @@ def reload_vae_weights(sd_model=None, vae_file=unspecified): lowvram.send_everything_to_cpu() else: sd_model.to(devices.cpu) - sd_hijack.model_hijack.undo_hijack(sd_model) - if shared.cmd_opts.rollback_vae and devices.dtype_vae == torch.bfloat16: - devices.dtype_vae = torch.float16 - load_vae(sd_model, vae_file, vae_source) - sd_hijack.model_hijack.hijack(sd_model) - script_callbacks.model_loaded_callback(sd_model) + + if shared.backend == shared.Backend.ORIGINAL: + sd_hijack.model_hijack.undo_hijack(sd_model) + if shared.cmd_opts.rollback_vae and devices.dtype_vae == torch.bfloat16: + devices.dtype_vae = torch.float16 + load_vae(sd_model, vae_file, vae_source) + sd_hijack.model_hijack.hijack(sd_model) + script_callbacks.model_loaded_callback(sd_model) + elif shared.backend == shared.Backend.DIFFUSERS: + load_vae_diffusers(sd_model, vae_file, vae_source) + if not shared.cmd_opts.lowvram and not shared.cmd_opts.medvram: sd_model.to(devices.device) shared.log.info(f"VAE weights loaded: {vae_file}") diff --git a/modules/shared.py b/modules/shared.py index 7d5b939d3..959ab7851 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -4,6 +4,7 @@ import time import json import datetime import urllib.request +from urllib.parse import urlparse from enum import Enum import gradio as gr import tqdm @@ -37,6 +38,10 @@ hypernetworks = {} loaded_hypernetworks = [] gradio_theme = gr.themes.Base() settings_components = None +pipelines = [ + 'Stable Diffusion', 'Stable Diffusion XL', 'Kandinsky V1', 'Kandinsky V2', 'DeepFloyd IF', 'Shap-E', + 'Stable Diffusion Img2Img', 'Stable Diffusion XL Img2Img', 'Kandinsky V1 Img2Img', 'Kandinsky V2 Img2Img', 'DeepFloyd IF Img2Img', 'Shap-E Img2Img' +] latent_upscale_default_mode = "Latent" latent_upscale_modes = { "Latent": {"mode": "bilinear", "antialias": False}, @@ -72,6 +77,11 @@ ui_reorder_categories = [ ] +def is_url(string): + parsed_url = urlparse(string) + return all([parsed_url.scheme, parsed_url.netloc]) + + class Backend(Enum): ORIGINAL = 1 DIFFUSERS = 2 @@ -182,10 +192,12 @@ class State: state = State() state.server_start = time.time() +backend = Backend.DIFFUSERS if (cmd_opts.backend is not None) and (cmd_opts.backend.lower() == 'diffusers') else Backend.ORIGINAL +log.info(f'Pipeline: {backend}') class OptionInfo: - def __init__(self, default=None, label="", component=None, component_args=None, onchange=None, section=None, refresh=None, comment_before='', comment_after=''): + def __init__(self, default=None, label="", component=None, component_args=None, onchange=None, section=None, refresh=None, submit=None, comment_before='', comment_after=''): self.default = default self.label = label self.component = component @@ -195,6 +207,7 @@ class OptionInfo: self.refresh = refresh self.comment_before = comment_before # HTML text that will be added after label in UI self.comment_after = comment_after # HTML text that will be added before label in UI + self.submit = submit def link(self, label, uri): self.comment_before += f"[{label}]" @@ -208,6 +221,10 @@ class OptionInfo: self.comment_after += f"({info})" return self + def html(self, info): + self.comment_after += f"{info}" + return self + def needs_restart(self): self.comment_after += " (requires restart)" return self @@ -223,7 +240,6 @@ def list_checkpoint_tiles(): import modules.sd_models # pylint: disable=W0621 return modules.sd_models.checkpoint_tiles() - default_checkpoint = list_checkpoint_tiles()[0] if len(list_checkpoint_tiles()) > 0 else "model.ckpt" @@ -251,12 +267,17 @@ def list_themes(): return themes -def lora_disable(): +def disable_extensions(): if opts.lora_disable: if 'Lora' not in opts.disabled_extensions: opts.data['disabled_extensions'].append('Lora') else: opts.data['disabled_extensions'] = [x for x in opts.disabled_extensions if x != 'Lora'] + if backend == Backend.DIFFUSERS: + for ext in ['sd-webui-controlnet', 'sd-dynamic-thresholding', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris']: + if ext not in opts.disabled_extensions: + log.warning(f'Diffusers disabling uncompatible extension: {ext}') + opts.data['disabled_extensions'].append(ext) def refresh_themes(): @@ -287,8 +308,9 @@ else: # cuda cross_attention_optimization_default ="Scaled-Dot-Product" options_templates.update(options_section(('sd', "Stable Diffusion"), { - "sd_model_checkpoint": OptionInfo(default_checkpoint, "Stable Diffusion checkpoint", gr.Dropdown, lambda: {"choices": list_checkpoint_tiles()}, refresh=refresh_checkpoints), "sd_checkpoint_autoload": OptionInfo(True, "Stable Diffusion checkpoint autoload on server start"), + "sd_model_checkpoint": OptionInfo(default_checkpoint, "Stable Diffusion checkpoint", gr.Dropdown, lambda: {"choices": list_checkpoint_tiles()}, refresh=refresh_checkpoints), + "sd_model_refiner": OptionInfo('None', "Stable Diffusion refiner", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints), "sd_checkpoint_cache": OptionInfo(0, "Number of cached model checkpoints", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}), "sd_vae_checkpoint_cache": OptionInfo(0, "Number of cached VAE checkpoints", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}), "sd_vae": OptionInfo("Automatic", "Select VAE", gr.Dropdown, lambda: {"choices": shared_items.sd_vae_items()}, refresh=shared_items.refresh_vae_list), @@ -299,7 +321,7 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), { "prompt_mean_norm": OptionInfo(True, "Prompt attention mean normalization"), "comma_padding_backtrack": OptionInfo(20, "Prompt padding for long prompts", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }), "sd_disable_ckpt": OptionInfo(False, "Disallow usage of checkpoints in ckpt format"), - "sd_backend": OptionInfo("Original", "Stable Diffusion backend (experimental)", gr.Radio, lambda: {"choices": ["Original", "Diffusers"] }), + "sd_backend": OptionInfo("original", "Stable Diffusion backend (experimental)", gr.Radio, lambda: {"choices": ["original", "diffusers"] }), })) options_templates.update(options_section(('optimizations', "Optimizations"), { @@ -329,10 +351,23 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "cuda_allow_tf32": OptionInfo(True, "Allow TF32 math ops"), "cuda_allow_tf16_reduced": OptionInfo(True, "Allow TF16 reduced precision math ops"), "cuda_compile": OptionInfo(False, "Enable model compile (experimental)"), - "cuda_compile_mode": OptionInfo("none", "Model compile mode (experimental)", gr.Radio, lambda: {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex']}), + "cuda_compile_mode": OptionInfo("none", "Model compile mode (experimental)", gr.Radio, lambda: {"choices": ['none', 'inductor', 'reduce-overhead', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex']}), + "cuda_compile_fullgraph": OptionInfo(False, "Model compile fullgraph"), "cuda_compile_verbose": OptionInfo(False, "Model compile verbose mode"), "cuda_compile_errors": OptionInfo(True, "Model compile suppress errors"), - "disable_gc": OptionInfo(False, "Disable Torch memory garbage collection"), + "disable_gc": OptionInfo(True, "Disable Torch memory garbage collection on each generation"), +})) + +options_templates.update(options_section(('diffusers', "Diffusers Settings"), { + "diffusers_allow_safetensors": OptionInfo(False, 'Diffuser Pipeline when loading from safetensors'), + "diffusers_pipeline": OptionInfo(pipelines[0], 'Diffuser Pipeline when loading from safetensors', gr.Dropdown, lambda: {"choices": pipelines}), + "diffusers_extract_ema": OptionInfo(True, "Use model EMA weights when possible"), + "diffusers_generator_device": OptionInfo("default", "Generator device", gr.Radio, lambda: {"choices": ["default", "cpu"]}), + "diffusers_seq_cpu_offload": OptionInfo(False, "Enable sequential CPU offload"), + "diffusers_model_cpu_offload": OptionInfo(False, "Enable model CPU offload"), + "diffusers_vae_slicing": OptionInfo(False, "Enable VAE slicing"), + "diffusers_vae_tiling": OptionInfo(False, "Enable VAE tiling"), + "diffusers_attention_slicing": OptionInfo(False, "Enable attention slicing"), })) options_templates.update(options_section(('system-paths', "System Paths"), { @@ -401,7 +436,7 @@ options_templates.update(options_section(('image-processing', "Image Processing" })) -options_templates.update(options_section(('saving-paths', "Output Paths"), { +options_templates.update(options_section(('saving-paths', "Image Paths"), { "outdir_samples": OptionInfo("", "Output directory for images", component_args=hide_dirs), "outdir_txt2img_samples": OptionInfo("outputs/text", 'Output directory for txt2img images', component_args=hide_dirs), "outdir_img2img_samples": OptionInfo("outputs/image", 'Output directory for img2img images', component_args=hide_dirs), @@ -432,7 +467,6 @@ options_templates.update(options_section(('ui', "User Interface"), { "ui_tab_reorder": OptionInfo("From Text, From Image, Process Image", "UI tabs order"), "ui_scripts_reorder": OptionInfo("Enable Dynamic Thresholding, ControlNet", "UI scripts order"), "ui_reorder": OptionInfo(", ".join(ui_reorder_categories), "txt2img/img2img UI item order"), - "ui_extra_networks_tab_reorder": OptionInfo("Checkpoints, Lora, LyCORIS, Textual Inversion, Hypernetworks", "Extra networks tab order"), })) options_templates.update(options_section(('live-preview', "Live Previews"), { @@ -449,27 +483,59 @@ options_templates.update(options_section(('live-preview', "Live Previews"), { "logmonitor_refresh_period": OptionInfo(5000, "Log view update period, in milliseconds", gr.Slider, {"minimum": 0, "maximum": 30000, "step": 25}), })) + options_templates.update(options_section(('sampler-params', "Sampler Settings"), { - "show_samplers": OptionInfo(["Euler a", "UniPC", "DDIM", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), + "show_samplers": OptionInfo(["Euler a", "UniPC", "DEIS", "DDIM", "DPM 1S", "DPM 2M", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), "fallback_sampler": OptionInfo("Euler a", "Secondary sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), "force_latent_sampler": OptionInfo("None", "Force latent upscaler sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), - "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems"), - "enable_quantization": OptionInfo(True, "Enable samplers quantization for sharper and cleaner results"), - "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - "eta_ddim": OptionInfo(0.0, "Noise multiplier for DDIM (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - "ddim_discretize": OptionInfo('uniform', "DDIM discretize img2img", gr.Radio, {"choices": ['uniform', 'quad']}), - 's_churn': OptionInfo(0.0, "sigma churn", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - 's_min_uncond': OptionInfo(0, "sigma negative guidance minimum ", gr.Slider, {"minimum": 0.0, "maximum": 4.0, "step": 0.01}), - 's_tmin': OptionInfo(0.0, "sigma tmin", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - 's_noise': OptionInfo(1.0, "sigma noise", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - 'eta_noise_seed_delta': OptionInfo(0, "Noise seed delta (eta)", gr.Number, {"precision": 0}), - 'always_discard_next_to_last_sigma': OptionInfo(False, "Always discard next-to-last sigma"), - 'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}), - 'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}), - 'uni_pc_order': OptionInfo(3, "UniPC order (must be < sampling steps)", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), - 'uni_pc_lower_order_final': OptionInfo(True, "UniPC lower order final"), })) +if backend == Backend.ORIGINAL: + options_templates.update(options_section(('sampler-params', "Sampler Settings"), { + "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems"), + "enable_quantization": OptionInfo(True, "Enable samplers quantization for sharper and cleaner results"), + "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + "eta_ddim": OptionInfo(0.0, "Noise multiplier for DDIM (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + "ddim_discretize": OptionInfo('uniform', "DDIM discretize img2img", gr.Radio, {"choices": ['uniform', 'quad']}), + 's_churn': OptionInfo(0.0, "sigma churn", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + 's_min_uncond': OptionInfo(0, "sigma negative guidance minimum ", gr.Slider, {"minimum": 0.0, "maximum": 4.0, "step": 0.01}), + 's_tmin': OptionInfo(0.0, "sigma tmin", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + 's_noise': OptionInfo(1.0, "sigma noise", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + 'eta_noise_seed_delta': OptionInfo(0, "Noise seed delta (eta)", gr.Number, {"precision": 0}), + 'always_discard_next_to_last_sigma': OptionInfo(False, "Always discard next-to-last sigma"), + 'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}), + 'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}), + 'uni_pc_order': OptionInfo(3, "UniPC order (must be < sampling steps)", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), + 'uni_pc_lower_order_final': OptionInfo(True, "UniPC lower order final"), + })) +elif backend == Backend.DIFFUSERS: + options_templates.update(options_section(('sampler-params', "Sampler Settings"), { + # hidden - included for compatibility only + "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching enabled on low memory systems", gr.Checkbox, { "visible": False}), + "enable_quantization": OptionInfo(True, "Enable samplers quantization for sharper and cleaner results", gr.Checkbox, { "visible": False}), + "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Number, { "visible": False}), + "eta_ddim": OptionInfo(0.0, "Noise multiplier for DDIM (eta)", gr.Number, { "visible": False}), + "ddim_discretize": OptionInfo('uniform', "", gr.Text, { "visible": False}), + 's_churn': OptionInfo(0.0, "sigma churn", gr.Number, { "visible": False}), + 's_min_uncond': OptionInfo(0, "sigma negative guidance minimum ", gr.Number, { "visible": False}), + 's_tmin': OptionInfo(0.0, "sigma tmin", gr.Number, { "visible": False}), + 's_noise': OptionInfo(1.0, "sigma noise", gr.Number, { "visible": False}), + 'eta_noise_seed_delta': OptionInfo(0, "Noise seed delta (eta)", gr.Number, {"precision": 0}, { "visible": False}), + 'always_discard_next_to_last_sigma': OptionInfo(False, "Always discard next-to-last sigma", gr.Checkbox, { "visible": False}), + #'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}, { "visible": False}), + #'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}, { "visible": False}), + #'uni_pc_order': OptionInfo(3, "UniPC order (must be < sampling steps)", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}, { "visible": False}), + #'uni_pc_lower_order_final': OptionInfo(True, "UniPC lower order final", { "visible": False}), + # diffuser specific + "schedulers_prediction_type": OptionInfo("default", "Samplers override model prediction type", gr.Radio, lambda: {"choices": ['default', 'epsilon', 'sample', 'v-prediction']}), + "schedulers_beta_schedule": OptionInfo("default", "Samplers override beta schedule", gr.Radio, lambda: {"choices": ['default', 'linear', 'scaled_linear', 'squaredcos_cap_v2']}), + "schedulers_solver_order": OptionInfo(2, "Samplers solver order where applicable", gr.Slider, {"minimum": 1, "maximum": 5, "step": 1}), + "schedulers_use_karras": OptionInfo(True, "Samplers should use Karras sigmas where applicable"), + "schedulers_use_loworder": OptionInfo(True, "Samplers should use use lower-order solvers in the final steps where applicable"), + "schedulers_use_thresholding": OptionInfo(False, "Samplers should use dynamic thresholding where applicable"), + "schedulers_dpm_solver": OptionInfo("sde-dpmsolver++", "Samplers DPM solver algorithm", gr.Radio, lambda: {"choices": ['dpmsolver', 'dpmsolver++', 'sde-dpmsolver++']}), + })) + options_templates.update(options_section(('postprocessing', "Postprocessing"), { 'postprocessing_enable_in_main_ui': OptionInfo([], "Enable addtional postprocessing operations", ui_components.DropdownMulti, lambda: {"choices": [x.name for x in shared_items.postprocessing_scripts()]}), 'postprocessing_operation_order': OptionInfo([], "Postprocessing operation order", ui_components.DropdownMulti, lambda: {"choices": [x.name for x in shared_items.postprocessing_scripts()]}), @@ -521,13 +587,17 @@ options_templates.update(options_section(('upscaling', "Upscaling"), { })) options_templates.update(options_section(('extra_networks', "Extra Networks"), { + "ui_extra_networks_tab_reorder": OptionInfo("Checkpoints, Lora, LyCORIS, Textual Inversion, Hypernetworks", "Extra networks tab order"), "extra_networks_card_cover": OptionInfo("inline", "UI position", gr.Radio, lambda: {"choices": ["cover", "inline", "sidebar"]}), + "extra_networks_height": OptionInfo(47, "UI height (%)", gr.Slider, {"minimum": 10, "maximum": 100, "step": 1}), "extra_networks_sidebar_width": OptionInfo(35, "UI sidebar width (%)", gr.Slider, {"minimum": 10, "maximum": 80, "step": 1}), + "extra_networks_card_lazy": OptionInfo(True, "UI card preview lazy loading"), "extra_networks_card_size": OptionInfo(200, "UI card size (px)", gr.Slider, {"minimum": 20, "maximum": 2000, "step": 1}), - "extra_networks_card_square": OptionInfo(False, "UI disable variable aspect ratio"), + "extra_networks_card_square": OptionInfo(True, "UI disable variable aspect ratio"), "extra_networks_card_fit": OptionInfo("cover", "UI image contain method", gr.Radio, lambda: {"choices": ["contain", "cover", "fill"]}), + "extra_network_skip_indexing": OptionInfo(False, "Do not automatically build extra network pages", gr.Checkbox), "lyco_patch_lora": OptionInfo(False, "Use LyCoris handler for all Lora types", gr.Checkbox), - "lora_disable": OptionInfo(False, "Disable built-in Lora handler", gr.Checkbox, { "visible": True }, onchange=lora_disable), + "lora_disable": OptionInfo(False, "Disable built-in Lora handler", gr.Checkbox, { "visible": True }, onchange=disable_extensions), "lora_functional": OptionInfo(False, "Use Kohya method for handling multiple Loras", gr.Checkbox), "extra_networks_add_text_separator": OptionInfo(" ", "Extra text to add before <...> when adding extra network to prompt", gr.Text, { "visible": False }), "extra_networks_default_multiplier": OptionInfo(1.0, "Multiplier for extra networks", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), @@ -682,13 +752,6 @@ opts = Options() config_filename = cmd_opts.config opts.load(config_filename) cmd_opts = cmd_args.compatibility_args(opts, cmd_opts) -if cmd_opts.backend == 'diffusers': - log.info('Overriding backend to Diffusers') - opts.data['sd_backend'] = 'Diffusers' -if cmd_opts.backend == 'original': - log.info('Overriding backend to Diffusers') - opts.data['sd_backend'] = 'Original' -backend = Backend.DIFFUSERS if opts.sd_backend == 'Diffusers' else Backend.ORIGINAL prompt_styles = modules.styles.StyleDatabase(opts.styles_dir) cmd_opts.disable_extension_access = (cmd_opts.share or cmd_opts.listen or (cmd_opts.server_name or False)) and not cmd_opts.insecure @@ -860,7 +923,6 @@ class Shared(sys.modules[__name__].__class__): @property def sd_model(self): import modules.sd_models # pylint: disable=W0621 - # return modules.sd_models.model_data.sd_model return modules.sd_models.model_data.get_sd_model() @sd_model.setter @@ -868,6 +930,17 @@ class Shared(sys.modules[__name__].__class__): import modules.sd_models # pylint: disable=W0621 modules.sd_models.model_data.set_sd_model(value) -# sd_model: LatentDiffusion = None # this var is here just for IDE's type checking; it cannot be accessed because the class field above will be accessed instead + @property + def sd_refiner(self): + import modules.sd_models # pylint: disable=W0621 + return modules.sd_models.model_data.get_sd_refiner() + + @sd_refiner.setter + def sd_refiner(self, value): + import modules.sd_models # pylint: disable=W0621 + modules.sd_models.model_data.set_sd_refiner(value) + + sd_model = None +sd_refiner = None sys.modules[__name__].__class__ = Shared diff --git a/modules/textual_inversion/image_embedding.py b/modules/textual_inversion/image_embedding.py index 7ef9b0d68..aca25bc9a 100644 --- a/modules/textual_inversion/image_embedding.py +++ b/modules/textual_inversion/image_embedding.py @@ -42,7 +42,7 @@ def lcg(m=2**32, a=1664525, c=1013904223, seed=0): def xor_block(block): g = lcg() - randblock = np.array([next(g) for _ in range(np.product(block.shape))]).astype(np.uint8).reshape(block.shape) + randblock = np.array([next(g) for _ in range(np.prod(block.shape))]).astype(np.uint8).reshape(block.shape) return np.bitwise_xor(block.astype(np.uint8), randblock & 0x0F) diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index ddb0fbe6d..c13e027c4 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -113,33 +113,56 @@ class EmbeddingDatabase: def register_embedding(self, embedding, model): self.word_embeddings[embedding.name] = embedding - ids = model.cond_stage_model.tokenize([embedding.name])[0] - first_id = ids[0] if first_id not in self.ids_lookup: self.ids_lookup[first_id] = [] - self.ids_lookup[first_id] = sorted(self.ids_lookup[first_id] + [(ids, embedding)], key=lambda x: len(x[0]), reverse=True) - return embedding def get_expected_shape(self): if shared.sd_model is None: shared.log.error('Model not loaded') return 0 + if shared.backend == shared.Backend.DIFFUSERS: + return 0 vec = shared.sd_model.cond_stage_model.encode_embedding_init_text(",", 1) return vec.shape[1] + def load_diffusers_embedding(self, filename: str, path: str): + fn, ext = os.path.splitext(filename) + if ext.lower() != ".pt" and ext.lower() != ".safetensors": + return + pipe = shared.sd_model + if filename == "": + pipe.tokenizer = pipe.tokenizer.__class__.from_pretrained(pipe.tokenizer.name_or_path) + pipe.text_encoder.resize_token_embeddings(len(pipe.tokenizer)) + return + name = os.path.basename(fn) + embedding = Embedding(vec=None, name=name) + try: + pipe.load_textual_inversion(path, cache_dir=shared.opts.data["diffusers_dir"], local_files_only=True) + self.word_embeddings[name] = embedding + except Exception: + self.skipped_embeddings[name] = embedding + try: + text_inv_tokens = pipe.tokenizer.added_tokens_encoder.keys() + text_inv_tokens = [t for t in text_inv_tokens if not (len(t.split("_")) > 1 and t.split("_")[-1].isdigit())] + except Exception: + text_inv_tokens = [] + pass + def load_from_file(self, path, filename): name, ext = os.path.splitext(filename) ext = ext.upper() + if shared.backend == shared.Backend.DIFFUSERS: + self.load_diffusers_embedding(filename, path) + return if ext in ['.PNG', '.WEBP', '.JXL', '.AVIF']: _, second_ext = os.path.splitext(name) if second_ext.upper() == '.PREVIEW': return - embed_image = Image.open(path) if hasattr(embed_image, 'text') and 'sd-ti-embedding' in embed_image.text: data = embedding_from_b64(embed_image.text['sd-ti-embedding']) @@ -206,15 +229,12 @@ class EmbeddingDatabase: continue def load_textual_inversion_embeddings(self, force_reload=False): - if shared.backend == shared.Backend.DIFFUSERS: # TODO Diffusers - return if not force_reload: need_reload = False for embdir in self.embedding_dirs.values(): if embdir.has_changed(): need_reload = True break - if not need_reload: return @@ -241,32 +261,25 @@ class EmbeddingDatabase: def find_embedding_at_position(self, tokens, offset): token = tokens[offset] possible_matches = self.ids_lookup.get(token, None) - if possible_matches is None: return None, None - for ids, embedding in possible_matches: if tokens[offset:offset + len(ids)] == ids: return embedding, len(ids) - return None, None def create_embedding(name, num_vectors_per_token, overwrite_old, init_text='*'): cond_model = shared.sd_model.cond_stage_model - with devices.autocast(): cond_model([""]) # will send cond model to GPU if lowvram/medvram is active - #cond_model expects at least some text, so we provide '*' as backup. embedded = cond_model.encode_embedding_init_text(init_text or '*', num_vectors_per_token) vec = torch.zeros((num_vectors_per_token, embedded.shape[1]), device=devices.device) - #Only copy if we provided an init_text, otherwise keep vectors as zeros if init_text: for i in range(num_vectors_per_token): vec[i] = embedded[i * int(embedded.shape[0]) // num_vectors_per_token] - # Remove illegal characters from name. name = "".join( x for x in name if (x.isalnum() or x in "._- ")) fn = os.path.join(shared.opts.embeddings_dir, f"{name}.pt") @@ -299,29 +312,33 @@ def write_loss(log_directory, filename, step, epoch_len, values): **values, }) + def tensorboard_setup(log_directory): os.makedirs(os.path.join(log_directory, "tensorboard"), exist_ok=True) return SummaryWriter( log_dir=os.path.join(log_directory, "tensorboard"), flush_secs=shared.opts.training_tensorboard_flush_every) + def tensorboard_add(tensorboard_writer, loss, global_step, step, learn_rate, epoch_num): tensorboard_add_scaler(tensorboard_writer, "Loss/train", loss, global_step) tensorboard_add_scaler(tensorboard_writer, f"Loss/train/epoch-{epoch_num}", loss, step) tensorboard_add_scaler(tensorboard_writer, "Learn rate/train", learn_rate, global_step) tensorboard_add_scaler(tensorboard_writer, f"Learn rate/train/epoch-{epoch_num}", learn_rate, step) + def tensorboard_add_scaler(tensorboard_writer, tag, value, step): tensorboard_writer.add_scalar(tag=tag, scalar_value=value, global_step=step) + def tensorboard_add_image(tensorboard_writer, tag, pil_image, step): # Convert a pil image to a torch tensor img_tensor = torch.as_tensor(np.array(pil_image, copy=True)) img_tensor = img_tensor.view(pil_image.size[1], pil_image.size[0], len(pil_image.getbands())) img_tensor = img_tensor.permute((2, 0, 1)) - tensorboard_writer.add_image(tag, img_tensor, global_step=step) + def validate_train_inputs(model_name, learn_rate, batch_size, gradient_step, data_root, template_file, template_filename, steps, save_model_every, create_image_every, log_directory, name="embedding"): assert model_name, f"{name} not selected" assert learn_rate, "Learning rate is empty or 0" @@ -383,15 +400,12 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st images_embeds_dir = None hijack = sd_hijack.model_hijack - embedding = hijack.embedding_db.word_embeddings[embedding_name] checkpoint = sd_models.select_checkpoint() - initial_step = embedding.step or 0 if initial_step >= steps: shared.state.textinfo = "Model has already been trained beyond specified max steps" return embedding, filename - scheduler = LearnRateScheduler(learn_rate, steps, initial_step) clip_grad = torch.nn.utils.clip_grad_value_ if clip_grad_mode == "value" else \ torch.nn.utils.clip_grad_norm_ if clip_grad_mode == "norm" else \ diff --git a/modules/ui.py b/modules/ui.py index 3edab8f2b..3de32c239 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -13,7 +13,7 @@ from modules.call_queue import wrap_gradio_gpu_call, wrap_queued_call, wrap_grad from modules import sd_hijack, sd_models, script_callbacks, ui_extensions, deepbooru, extra_networks, ui_common, ui_postprocessing, ui_loadsave, ui_train, ui_models from modules.ui_components import FormRow, FormColumn, FormGroup, ToolButton, FormHTML # pylint: disable=unused-import from modules.paths import script_path, data_path -from modules.shared import opts, cmd_opts, backend, Backend +from modules.shared import opts, cmd_opts from modules import prompt_parser import modules.codeformer_model import modules.generation_parameters_copypaste as parameters_copypaste @@ -198,13 +198,21 @@ def update_token_counter(text, steps): prompt_schedules = [[[steps, text]]] flat_prompts = reduce(lambda list1, list2: list1+list2, prompt_schedules) prompts = [prompt_text for step, prompt_text in flat_prompts] - if backend == Backend.ORIGINAL: + if modules.shared.backend == modules.shared.Backend.ORIGINAL: token_count, max_length = max([sd_hijack.model_hijack.get_prompt_lengths(prompt) for prompt in prompts], key=lambda args: args[0]) - else: - tokenizer = modules.shared.sd_model.tokenizer - has_bos_token, has_eos_token = tokenizer.bos_token_id is not None, tokenizer.eos_token_id is not None - token_count = max([len(modules.shared.sd_model.tokenizer(prompt)) for prompt in prompts]) - int(has_bos_token) - int(has_eos_token) - max_length = tokenizer.model_max_length - int(has_bos_token) - int(has_eos_token) + elif modules.shared.backend == modules.shared.Backend.DIFFUSERS: + if modules.shared.sd_model is not None: + tokenizer = modules.shared.sd_model.tokenizer + has_bos_token = tokenizer.bos_token_id is not None + has_eos_token = tokenizer.eos_token_id is not None + ids = [modules.shared.sd_model.tokenizer(prompt) for prompt in prompts] + if len(ids) > 0 and hasattr(ids[0], 'input_ids'): + ids = [x.input_ids for x in ids] + token_count = max([len(x) for x in ids]) - int(has_bos_token) - int(has_eos_token) + max_length = tokenizer.model_max_length - int(has_bos_token) - int(has_eos_token) + else: + token_count = 0 + max_length = 75 return f"{token_count}/{max_length}" @@ -321,7 +329,10 @@ def create_override_settings_dropdown(tabname, row): # pylint: disable=unused-ar return dropdown -def create_ui(): +def create_ui(startup_timer = None): + if startup_timer is None: + from modules import timer + startup_timer = timer.Timer() import modules.img2img # pylint: disable=redefined-outer-name import modules.txt2img # pylint: disable=redefined-outer-name reload_javascript() @@ -334,7 +345,7 @@ def create_ui(): txt_prompt_img = gr.File(label="", elem_id="txt2img_prompt_image", file_count="single", type="binary", visible=False) with FormRow(variant='compact', elem_id="txt2img_extra_networks", visible=False) as extra_networks_ui: from modules import ui_extra_networks - extra_networks_ui = ui_extra_networks.create_ui(extra_networks_ui, extra_networks_button, 'txt2img') + extra_networks_ui = ui_extra_networks.create_ui(extra_networks_ui, extra_networks_button, 'txt2img', skip_indexing=opts.extra_network_skip_indexing) with gr.Row().style(equal_height=False, elem_id="txt2img_interface"): with gr.Column(variant='compact', elem_id="txt2img_settings"): for category in ordered_ui_categories(): @@ -492,9 +503,10 @@ def create_ui(): ui_extra_networks.setup_ui(extra_networks_ui, txt2img_gallery) + startup_timer.record("ui-txt2img") + modules.scripts.scripts_current = modules.scripts.scripts_img2img modules.scripts.scripts_img2img.initialize_scripts(is_img2img=True) - with gr.Blocks(analytics_enabled=False) as img2img_interface: img2img_prompt, img2img_prompt_styles, img2img_negative_prompt, submit, img2img_interrogate, img2img_deepbooru, img2img_prompt_style_apply, img2img_save_style, img2img_paste, extra_networks_button, token_counter, token_button, negative_token_counter, negative_token_button = create_toprow(is_img2img=True) @@ -502,7 +514,7 @@ def create_ui(): with FormRow(variant='compact', elem_id="img2img_extra_networks", visible=False) as extra_networks_ui: from modules import ui_extra_networks - extra_networks_ui_img2img = ui_extra_networks.create_ui(extra_networks_ui, extra_networks_button, 'img2img') + extra_networks_ui_img2img = ui_extra_networks.create_ui(extra_networks_ui, extra_networks_button, 'img2img', skip_indexing=opts.extra_network_skip_indexing) with FormRow().style(equal_height=False, elem_id="img2img_interface"): with gr.Column(variant='compact', elem_id="img2img_settings"): @@ -849,16 +861,21 @@ def create_ui(): paste_button=img2img_paste, tabname="img2img", source_text_component=img2img_prompt, source_image_component=None, )) + startup_timer.record("ui-img2img") + modules.scripts.scripts_current = None with gr.Blocks(analytics_enabled=False) as extras_interface: ui_postprocessing.create_ui() + startup_timer.record("ui-extras") with gr.Blocks(analytics_enabled=False) as train_interface: ui_train.create_ui(txt2img_preview_params = [txt2img_prompt, txt2img_negative_prompt, steps, sampler_index, cfg_scale, seed, width, height]) + startup_timer.record("ui-train") with gr.Blocks(analytics_enabled=False) as models_interface: ui_models.create_ui() + startup_timer.record("ui-models") def create_setting_component(key, is_quicksettings=False): def fun(): @@ -897,9 +914,13 @@ def create_ui(): res = comp(label=info.label, value=fun(), elem_id=elem_id, **(args or {})) create_refresh_button(res, info.refresh, info.component_args, f"refresh_{key}") else: - res = comp(label=info.label, value=fun(), elem_id=elem_id, **(args or {})) + try: + res = comp(label=info.label, value=fun(), elem_id=elem_id, **(args or {})) + except Exception as e: + modules.shared.log.error(f'Error creating setting: {key} {e}') + res = None - if not is_quicksettings: + if res is not None and not is_quicksettings: res.change(fn=None, inputs=res, _js=f'(val) => markIfModified("{key}", val)') dirty_indicator.click(fn=lambda: getattr(opts, key), outputs=res, show_progress=False) dirtyable_setting.__exit__() @@ -971,6 +992,7 @@ def create_ui(): quicksettings_names = opts.quicksettings_list quicksettings_names = {x: i for i, x in enumerate(quicksettings_names) if x != 'quicksettings'} quicksettings_list = [] + previous_section = [] tab_item_keys = [] current_tab = None @@ -1017,7 +1039,8 @@ def create_ui(): create_dirty_indicator("show_all_pages", [], interactive=False) def unload_sd_weights(): - modules.sd_models.unload_model_weights() + modules.sd_models.unload_model_weights(op='model') + modules.sd_models.unload_model_weights(op='refiner') def reload_sd_weights(): modules.sd_models.reload_model_weights() @@ -1048,6 +1071,7 @@ def create_ui(): outputs=[dummy_component] ) + startup_timer.record("ui-settings") interfaces = [ (txt2img_interface, "From Text", "txt2img"), @@ -1060,6 +1084,7 @@ def create_ui(): interfaces += [(settings_interface, "Settings", "settings")] extensions_interface = ui_extensions.create_ui() interfaces += [(extensions_interface, "Extensions", "extensions")] + startup_timer.record("ui-extensions") modules.shared.tab_names = [] for _interface, label, _ifid in interfaces: @@ -1135,6 +1160,8 @@ def create_ui(): queue=False, ) + startup_timer.record("ui-defaults") + loadsave.dump_defaults() demo.ui_loadsave = loadsave @@ -1146,7 +1173,6 @@ def webpath(fn): web_path = os.path.relpath(fn, script_path).replace('\\', '/') else: web_path = os.path.abspath(fn) - return f'file={web_path}?{os.path.getmtime(fn)}' @@ -1171,12 +1197,7 @@ def html_body(): body = '' inline = '' if opts.theme_style != 'Auto': - if opts.gradio_theme == 'black-orange': - modules.shared.log.info('Theme does not support custom mode') - else: - inline += f"set_theme('{opts.theme_style.lower()}');" - if opts.gradio_theme == 'black-orange': - inline += "set_theme('dark');" + inline += f"set_theme('{opts.theme_style.lower()}');" body += f'\n' return body diff --git a/modules/ui_extensions.py b/modules/ui_extensions.py index f5df622f5..519dcbb64 100644 --- a/modules/ui_extensions.py +++ b/modules/ui_extensions.py @@ -17,7 +17,7 @@ sort_ordering = { "default": (True, lambda x: x.get('sort_default', '')), "user extensions": (True, lambda x: x.get('sort_user', '')), "trending": (True, lambda x: x.get('sort_trending', -1)), - "update avilable": (True, lambda x: x.get('sort_update', '')), + "update available": (True, lambda x: x.get('sort_update', '')), "updated date": (True, lambda x: x.get('updated', '2000-01-01T00:00')), "created date": (True, lambda x: x.get('created', '2000-01-01T00:00')), "name": (False, lambda x: x.get('name', '').lower()), diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index ba4e7662e..e2756917c 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -66,6 +66,7 @@ class ExtraNetworksPage: self.allow_negative_prompt = False self.metadata = {} self.info = {} + self.html = '' self.items = [] self.missing_thumbs = [] self.card = ''' @@ -82,7 +83,7 @@ class ExtraNetworksPage: ℹ️ - + ''' # noqa: RUF001 @@ -133,25 +134,32 @@ class ExtraNetworksPage: fn = f'{fn}.thumb.jpg' if os.path.exists(fn): continue - created += 1 - img = Image.open(f) - img = img.convert('RGB') - img.thumbnail((512, 512), Image.HAMMING) - img.save(fn) - img.close() + try: + img = Image.open(f) + img = img.convert('RGB') + img.thumbnail((512, 512), Image.HAMMING) + img.save(fn) + img.close() + created += 1 + except Exception as e: + shared.log.error(f'Extra network error creating thumbnail: {f} {e}') if len(self.missing_thumbs) > 0: shared.log.info(f"Extra network created thumbnails: {self.name} {created}") self.missing_thumbs.clear() - def create_html(self, tabname): - items_html = '' + def create_html(self, tabname, skip = False): + self_name_id = self.name.replace(" ", "_") + if skip: + return f"
Extra network page not ready
Click refresh to try again
" subdirs = {} allowed_folders = [os.path.abspath(x) for x in self.allowed_directories_for_previews()] for parentdir in [*set(allowed_folders)]: for root, dirs, _files in os.walk(parentdir, followlinks=True): for dirname in dirs: x = os.path.join(root, dirname) - if not os.path.isdir(x): + if shared.opts.diffusers_dir in x: + subdirs[os.path.basename(shared.opts.diffusers_dir)] = 1 + if (not os.path.isdir(x)) or ('models--' in x): continue subdir = os.path.abspath(x)[len(parentdir):].replace("\\", "/") while subdir.startswith("/"): @@ -166,28 +174,26 @@ class ExtraNetworksPage: {html.escape(subdir) if subdir!="" else "all"}
""" for subdir in subdirs]) try: + if len(self.html) > 0: + res = f"
{subdirs_html}
{self.html}
" + return res + self.html = '' self.items = list(self.list_items()) self.create_xyz_grid() for item in self.items: self.metadata[item["name"]] = item.get("metadata", {}) self.info[item["name"]] = self.find_info(item['filename']) - items_html += self.create_html_for_item(item, tabname) - # if items_html == '': - # dirs = "".join([f"
  • {x}
    • " for x in self.allowed_directories_for_previews()]) - # items_html = f'
    No models found: {dirs}
    ' - self_name_id = self.name.replace(" ", "_") - if len(subdirs_html) > 0 or len(items_html) > 0: - res = f""" -
    {subdirs_html}
    -
    {items_html}
    - """ + self.html += self.create_html_for_item(item, tabname) + if len(subdirs_html) > 0 or len(self.html) > 0: + res = f"
    {subdirs_html}
    {self.html}
    " else: return '' + shared.log.debug(f'Extra networks: {self.name} items={len(self.items)} subdirs={len(subdirs)}') threading.Thread(target=self.create_thumb).start() return res except Exception as e: shared.log.error(f'Extra networks page error: {e}') - return '' + return f"
    Extra network error
    {e}
    " def list_items(self): raise NotImplementedError @@ -208,6 +214,7 @@ class ExtraNetworksPage: "name": item["name"], "description": (item.get("description") or ""), "search_term": item.get("search_term", ""), + "loading": "lazy" if shared.opts.extra_networks_card_lazy else "eager", "card_click": item.get("onclick", '"' + html.escape(f"""return cardClicked({json.dumps(tabname)}, {item.get("prompt", None)}, {"true" if self.allow_negative_prompt else "false"})""") + '"'), "card_save_desc": '"' + html.escape(f"""return saveCardDescription(event, {json.dumps(tabname)}, {json.dumps(item["local_preview"])})""") + '"', "card_save_preview": '"' + html.escape(f"""return saveCardPreview(event, {json.dumps(tabname)}, {json.dumps(item["local_preview"])})""") + '"', @@ -290,7 +297,7 @@ def sort_extra_pages(pages): return sorted(pages, key=lambda x: tab_scores[x.name]) -def create_ui(container, button, tabname): +def create_ui(container, button, tabname, skip_indexing = False): ui = ExtraNetworksUi() ui.pages = [] ui.stored_extra_pages = sort_extra_pages(extra_pages) @@ -308,7 +315,7 @@ def create_ui(container, button, tabname): ui.description_target_filename = gr.Textbox('Description save filename', elem_id=tabname+"_description_filename", visible=False) for page in ui.stored_extra_pages: - page_html = page.create_html(ui.tabname) + page_html = page.create_html(ui.tabname, skip_indexing) if len(page_html) > 0: with gr.Tab(page.title, id=page.title.lower().replace(" ", "_"), elem_classes="extra-networks-tab"): page_elem = gr.HTML(page_html, elem_id=tabname+page.name+"_extra_page", elem_classes="extra-networks-page") @@ -326,6 +333,7 @@ def create_ui(container, button, tabname): def refresh(): res = [] for pg in ui.stored_extra_pages: + pg.html = '' pg.refresh() res.append(pg.create_html(ui.tabname)) ui.search.update(value = ui.search.value) diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index fe65d99df..37bee332b 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -21,10 +21,10 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): "filename": path, "preview": self.find_preview(path), "description": self.find_description(path), - "search_term": self.search_terms_from_path(checkpoint.filename) + " " + (checkpoint.sha256 or ""), + "search_term": f'{self.search_terms_from_path(checkpoint.filename)} {(checkpoint.sha256 or "")} /{checkpoint.type}/', "onclick": '"' + html.escape(f"""return selectCheckpoint({json.dumps(name)})""") + '"', "local_preview": f"{path}.{shared.opts.samples_format}", } def allowed_directories_for_previews(self): - return [v for v in [shared.opts.ckpt_dir, sd_models.model_path] if v is not None] + return [v for v in [shared.opts.ckpt_dir, shared.opts.diffusers_dir, sd_models.model_path] if v is not None] diff --git a/modules/ui_extra_networks_hypernets.py b/modules/ui_extra_networks_hypernets.py index d29863212..01cb22c6c 100644 --- a/modules/ui_extra_networks_hypernets.py +++ b/modules/ui_extra_networks_hypernets.py @@ -19,7 +19,7 @@ class ExtraNetworksPageHypernetworks(ui_extra_networks.ExtraNetworksPage): "preview": self.find_preview(path), "description": self.find_description(path), "search_term": self.search_terms_from_path(path), - "prompt": json.dumps(f""), + "prompt": json.dumps(f""), "local_preview": f"{path}.preview.{shared.opts.samples_format}", } diff --git a/modules/ui_models.py b/modules/ui_models.py index aa1c082f6..592fb2e08 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -161,3 +161,63 @@ def create_ui(): return model_data, txt model_list_btn.click(fn=list_models, inputs=[], outputs=[model_table, models_outcome]) + + with gr.Tab(label="Huggingface"): + data = [] + os.environ.setdefault('HF_HUB_DISABLE_EXPERIMENTAL_WARNING', '1') + os.environ.setdefault('HF_HUB_DISABLE_SYMLINKS_WARNING', '1') + os.environ.setdefault('HF_HUB_DISABLE_IMPLICIT_TOKEN', '1') + os.environ.setdefault('HUGGINGFACE_HUB_VERBOSITY', 'warning') + + def hf_search(keyword): + import huggingface_hub as hf + hf_api = hf.HfApi() + model_filter = hf.ModelFilter( + model_name=keyword, + task='text-to-image', + library=['diffusers'], + ) + models = hf_api.list_models(filter=model_filter, full=True, limit=50, sort="downloads", direction=-1) + data.clear() + for model in models: + tags = [t for t in model.tags if not t.startswith('diffusers') and not t.startswith('license') and not t.startswith('arxiv') and len(t) > 2] + data.append([model.modelId, model.pipeline_tag, tags, model.downloads, model.lastModified, f'https://huggingface.co/{model.modelId}']) + return data + + def hf_select(evt: gr.SelectData, data): + return data[evt.index[0]][0] + + def hf_download_model(hub_id: str, token): + from modules.shared import log, opts + from modules.modelloader import download_diffusers_model + try: + download_diffusers_model(hub_id, cache_dir=opts.diffusers_dir, token=token) + except Exception as e: + log.error(f"Diffuser model downloaded error: model={hub_id} {e}") + return f"Diffuser model downloaded error: model={hub_id} {e}" + from modules.sd_models import list_models # pylint: disable=W0621 + list_models() + log.info(f"Diffuser model downloaded: model={hub_id}") + return f'Diffuser model downloaded: model={hub_id}' + + with gr.Column(scale=6): + with gr.Row(): + hf_search_text = gr.Textbox('', label = 'Seach models', placeholder='search huggingface models') + with gr.Row(): + hf_selected = gr.Textbox('', label = 'Select model', placeholder='select model from search results or enter model name manually') + with gr.Row(): + hf_token = gr.Textbox('', label = 'Huggingface token', placeholder='optional access token for private or gated models') + with gr.Column(scale=1): + hf_download_model_btn = gr.Button(value="Download model", variant='primary') + + with gr.Row(): + hf_headers = ['Name', 'Pipeline', 'Tags', 'Downloads', 'Updated', 'URL'] + hf_types = ['str', 'str', 'str', 'number', 'date', 'markdown'] + hf_results = gr.DataFrame([], label = 'Search results', show_label = True, interactive = False, wrap = True, overflow_row_behaviour = 'paginate', max_rows = 10, headers = hf_headers, datatype = hf_types, type='array') + + hf_search_text.submit(fn=hf_search, inputs=[hf_search_text], outputs=[hf_results]) + hf_results.select(fn=hf_select, inputs=[hf_results], outputs=[hf_selected]) + hf_download_model_btn.click(fn=hf_download_model, inputs=[hf_selected, hf_token], outputs=[models_outcome]) + + # with gr.Tab(label="CivitAI"): + # pass diff --git a/modules/ui_postprocessing.py b/modules/ui_postprocessing.py index b8a82d538..7f5ccc599 100644 --- a/modules/ui_postprocessing.py +++ b/modules/ui_postprocessing.py @@ -9,18 +9,16 @@ from modules.ui_common import infotext_to_html def wrap_pnginfo(image): _, geninfo, info = run_pnginfo(image) - return '', infotext_to_html(geninfo), info, geninfo + return infotext_to_html(geninfo), info, geninfo def submit_click(tab_index, extras_image, image_batch, extras_batch_input_dir, extras_batch_output_dir, show_extras_results, *script_inputs): - result_images, geninfo, js_info = postprocessing.run_postprocessing(tab_index, extras_image, image_batch, extras_batch_input_dir, extras_batch_output_dir, show_extras_results, *script_inputs) return result_images, geninfo, json.dumps(js_info), '' def create_ui(): tab_index = gr.State(value=0) # pylint: disable=abstract-class-instantiated - with gr.Row().style(equal_height=False, variant='compact'): with gr.Column(variant='compact'): with gr.Tabs(elem_id="mode_extras"): @@ -53,11 +51,10 @@ def create_ui(): tab_single.select(fn=lambda: 0, inputs=[], outputs=[tab_index]) tab_batch.select(fn=lambda: 1, inputs=[], outputs=[tab_index]) tab_batch_dir.select(fn=lambda: 2, inputs=[], outputs=[tab_index]) - _dummy = gr.HTML(visible=False) extras_image.change( fn=wrap_gradio_call(wrap_pnginfo), inputs=[extras_image], - outputs=[_dummy, html_info_formatted, exif_info, gen_info], + outputs=[html_info_formatted, exif_info, gen_info], ) submit.click( _js="submit_postprocessing", diff --git a/requirements.txt b/requirements.txt index 0aeebdd1a..59cc27e6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,6 @@ omegaconf open-clip-torch opencv-contrib-python piexif -Pillow psutil pyyaml realesrgan @@ -43,13 +42,14 @@ yapf scikit-image basicsr compel -typing-extensions==4.6.3 -pydantic==1.10.9 +typing-extensions==4.7.1 +antlr4-python3-runtime==4.9.3 +pydantic==1.10.11 requests==2.31.0 tqdm==4.65.0 accelerate==0.20.3 opencv-python==4.7.0.72 -diffusers==0.17.1 +diffusers==0.18.1 einops==0.4.1 gradio==3.32.0 numexpr==2.8.4 @@ -58,7 +58,8 @@ numba==0.57.0 pandas==1.5.3 protobuf==3.20.3 pytorch_lightning==1.9.4 -transformers==4.26.1 -timm==0.6.13 +transformers==4.30.2 tomesd==0.1.3 urllib3==1.26.15 +Pillow==9.5.0 +timm==0.6.13 diff --git a/scripts/outpainting_mk_2.py b/scripts/outpainting_mk_2.py index bf1daf55f..4b0af3053 100644 --- a/scripts/outpainting_mk_2.py +++ b/scripts/outpainting_mk_2.py @@ -99,7 +99,7 @@ def get_matched_noise(_np_src_image, np_mask_rgb, noise_q=1, color_variation=0.0 shaped_noise_fft = _fft2(noise_rgb) shaped_noise_fft[:, :, :] = np.absolute(shaped_noise_fft[:, :, :]) ** 2 * (src_dist ** noise_q) * src_phase # perform the actual shaping - brightness_variation = 0. # color_variation # todo: temporarily tieing brightness variation to color variation for now + brightness_variation = 0. # color_variation contrast_adjusted_np_src = _np_src_image[:] * (brightness_variation + 1.) - brightness_variation * 2. # scikit-image is used for histogram matching, very convenient! diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index 5ecb24134..11f716d00 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -410,9 +410,9 @@ class Script(scripts.Script): with gr.Row(variant="compact", elem_id="axis_options"): margin_size = gr.Slider(label="Grid margins", minimum=0, maximum=500, value=0, step=2, elem_id=self.elem_id("margin_size")) with gr.Row(variant="compact", elem_id="swap_axes"): - swap_xy_axes_button = gr.Button(value="Swap X/Y axes", elem_id="xy_grid_swap_axes_button") - swap_yz_axes_button = gr.Button(value="Swap Y/Z axes", elem_id="yz_grid_swap_axes_button") - swap_xz_axes_button = gr.Button(value="Swap X/Z axes", elem_id="xz_grid_swap_axes_button") + swap_xy_axes_button = gr.Button(value="Swap X/Y", elem_id="xy_grid_swap_axes_button", variant="secondary") + swap_yz_axes_button = gr.Button(value="Swap Y/Z", elem_id="yz_grid_swap_axes_button", variant="secondary") + swap_xz_axes_button = gr.Button(value="Swap X/Z", elem_id="xz_grid_swap_axes_button", variant="secondary") def swap_axes(axis1_type, axis1_values, axis1_values_dropdown, axis2_type, axis2_values, axis2_values_dropdown): return self.current_axis_options[axis2_type].label, axis2_values, axis2_values_dropdown, self.current_axis_options[axis1_type].label, axis1_values, axis1_values_dropdown diff --git a/webui.py b/webui.py index d95c290b8..1b8da00c7 100644 --- a/webui.py +++ b/webui.py @@ -101,6 +101,7 @@ def check_rollback_vae(): def initialize(): log.debug('Entering initialize') + shared.disable_extensions() check_rollback_vae() modules.sd_vae.refresh_vae_list() @@ -167,14 +168,18 @@ def load_model(): if opts.sd_checkpoint_autoload: shared.state.begin() shared.state.job = 'load model' - thread = Thread(target=lambda: shared.sd_model) - thread.start() + thread_model = Thread(target=lambda: shared.sd_model) + thread_model.start() + thread_refiner = Thread(target=lambda: shared.sd_refiner) + thread_refiner.start() shared.state.end() - thread.join() + thread_model.join() + thread_refiner.join() else: log.debug('Model auto load disabled') - shared.opts.onchange("sd_model_checkpoint", wrap_queued_call(lambda: modules.sd_models.reload_model_weights()), call=False) - shared.opts.onchange("sd_model_dict", wrap_queued_call(lambda: modules.sd_models.reload_model_weights()), call=False) + shared.opts.onchange("sd_model_checkpoint", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='model')), call=False) + shared.opts.onchange("sd_model_refiner", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='refiner')), call=False) + shared.opts.onchange("sd_model_dict", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='dict')), call=False) startup_timer.record("checkpoint") @@ -229,7 +234,7 @@ def start_ui(): log.debug('Creating UI') modules.script_callbacks.before_ui_callback() startup_timer.record("before-ui") - shared.demo = modules.ui.create_ui() + shared.demo = modules.ui.create_ui(startup_timer) startup_timer.record("ui") if cmd_opts.disable_queue: log.info('Server queues disabled') diff --git a/wiki b/wiki index f941746c0..e5da783be 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit f941746c0ed2afcaa37c1ec77b86da4dee131bee +Subproject commit e5da783bef6cacdb669c482d726e23a8d20d86d8