mirror of
https://github.com/vladmandic/automatic
synced 2026-09-19 17:24:32 +02:00
Vladimir Mandic
GitHub
@@ -1,5 +1,67 @@
|
||||
# Change Log for SD.Next
|
||||
|
||||
## Update for 2023-12-04
|
||||
|
||||
What's new? Native video in SD.Next via both **AnimateDiff** and **Stable-Video-Diffusion** - and including native MP4 encoding and smooth video outputs out-of-the-box, not just animated-GIFs.
|
||||
Also new is support for **SDXL-Turbo** as well as new **Kandinsky 3** models and cool latent correction via **HDR controls** for any *txt2img* workflows, best-of-class **SDXL model merge** using full ReBasin methods and further mobile UI optimizations.
|
||||
|
||||
- **Diffusers**
|
||||
- **IP adapter**
|
||||
- lightweight native implementation of T2I adapters which can guide generation towards specific image style
|
||||
- supports most T2I models, not limited to SD 1.5
|
||||
- models are auto-downloaded on first use
|
||||
- for IP adapter support in *Original* backend, use standard *ControlNet* extension
|
||||
- **AnimateDiff**
|
||||
- lightweight native implementation of AnimateDiff models:
|
||||
*AnimateDiff 1.4, 1.5 v1, 1.5 v2, AnimateFace*
|
||||
- supports SD 1.5 only
|
||||
- models are auto-downloaded on first use
|
||||
- for video saving support, see video support section
|
||||
- can be combined with IP-Adapter for even better results!
|
||||
- for AnimateDiff support in *Original* backend, use standard *AnimateDiff* extension
|
||||
- **HDR latent control**, based on [article](https://huggingface.co/blog/TimothyAlexisVass/explaining-the-sdxl-latent-space#long-prompts-at-high-guidance-scales-becoming-possible)
|
||||
- in *Advanced* params
|
||||
- allows control of *latent clamping*, *color centering* and *range maximimization*
|
||||
- supported by *XYZ grid*
|
||||
- [SD21 Turbo](https://huggingface.co/stabilityai/sd-turbo) and [SDXL Turbo](<https://huggingface.co/stabilityai/sdxl-turbo>) support
|
||||
- just set CFG scale (0.0-1.0) and steps (1-3) to a very low value
|
||||
- compatible with original StabilityAI SDXL-Turbo or any of the newer merges
|
||||
- download safetensors or select from networks -> reference
|
||||
- [Stable Video Diffusion](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid) and [Stable Video Diffusion XT](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid-xt) support
|
||||
- download using built-in model downloader or simply select from *networks -> reference*
|
||||
support for manually downloaded safetensors models will be added later
|
||||
- for video saving support, see video support section
|
||||
- go to *image* tab, enter input image and select *script* -> *stable video diffusion*
|
||||
- [Kandinsky 3](https://huggingface.co/kandinsky-community/kandinsky-3) support
|
||||
- download using built-in model downloader or simply select from *networks -> reference*
|
||||
- this model is absolutely massive at 27.5GB at fp16, so be patient
|
||||
- model params count is at 11.9B (compared to SD-XL at 3.3B) and its trained on mixed resolutions from 256px to 1024px
|
||||
- use either model offload or sequential cpu offload to be able to use it
|
||||
- better autodetection of *inpaint* and *instruct* pipelines
|
||||
- support long seconary prompt for refiner
|
||||
- **Video support**
|
||||
- applies to any model that supports video generation, e.g. AnimateDiff and StableVideoDiffusion
|
||||
- support for **animated-GIF**, **animated-PNG** and **MP4**
|
||||
- GIF and PNG can be looped
|
||||
- MP4 can have additional padding at the start/end as well as motion-aware interpolated frames for smooth playback
|
||||
interpolation is done using [RIFE](https://arxiv.org/abs/2011.06294) with native implementation in SD.Next
|
||||
And its fast - interpolation from 16 frames with 10x frames to target 160 frames results takes 2-3sec
|
||||
- output folder for videos is in *settings -> image paths -> video*
|
||||
- **General**
|
||||
- redesigned built-in profiler
|
||||
- now includes both `python` and `torch` and traces individual functions
|
||||
- use with `--debug --profile`
|
||||
- **model merge** add **SD-XL ReBasin** support, thanks @AI-Casanova
|
||||
- further UI optimizations for **mobile devices**, thanks @iDeNoh
|
||||
- log level defaults to info for console and debug for log file
|
||||
- better prompt display in process tab
|
||||
- increase maximum lora cache values
|
||||
- fix extra networks sorting
|
||||
- fix controlnet compatibility issues in original backend
|
||||
- fix img2img/inpaint paste params
|
||||
- fix save text file for manually saved images
|
||||
- fix python 3.9 compatibility issues
|
||||
|
||||
## Update for 2023-11-23
|
||||
|
||||
New release, primarily focused around three major new features: full **LCM** support, completely new **Model Merge** functionality and **Stable-fast** compile support
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ import diffusers
|
||||
import safetensors
|
||||
import safetensors.torch as sf
|
||||
|
||||
log = logging.getLogger("sdnext")
|
||||
log = logging.getLogger("sd")
|
||||
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s | %(message)s')
|
||||
|
||||
|
||||
|
||||
@@ -64,8 +64,9 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
|
||||
self.active = False
|
||||
|
||||
def deactivate(self, p):
|
||||
if shared.backend == shared.Backend.DIFFUSERS and hasattr(shared.sd_model, "unload_lora_weights"):
|
||||
shared.sd_model.unload_lora_weights()
|
||||
if shared.backend == shared.Backend.DIFFUSERS and hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"):
|
||||
if 'CLIP' in shared.sd_model.text_encoder.__class__.__name__ and not (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"):
|
||||
shared.sd_model.unload_lora_weights()
|
||||
if not self.active and getattr(networks, "originals", None ) is not None:
|
||||
networks.originals.undo() # remove patches
|
||||
if networks.debug:
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import os
|
||||
import torch
|
||||
import networks
|
||||
from modules import patches, shared
|
||||
|
||||
# OpenVINO only works with Diffusers LoRa loading
|
||||
force_lora_diffusers = os.environ.get('SD_LORA_DIFFUSERS', None) is not None
|
||||
|
||||
class LoraPatches:
|
||||
def __init__(self):
|
||||
@@ -18,7 +21,7 @@ class LoraPatches:
|
||||
self.MultiheadAttention_load_state_dict = None
|
||||
|
||||
def apply(self):
|
||||
if self.active or (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"): # OpenVINO only works with Diffusers LoRa loading
|
||||
if self.active or force_lora_diffusers:
|
||||
return
|
||||
self.Linear_forward = patches.patch(__name__, torch.nn.Linear, 'forward', networks.network_Linear_forward)
|
||||
self.Linear_load_state_dict = patches.patch(__name__, torch.nn.Linear, '_load_from_state_dict', networks.network_Linear_load_state_dict)
|
||||
@@ -36,7 +39,7 @@ class LoraPatches:
|
||||
self.active = True
|
||||
|
||||
def undo(self):
|
||||
if not self.active or (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"): # OpenVINO only works with Diffusers LoRa loading
|
||||
if not self.active or force_lora_diffusers:
|
||||
return
|
||||
self.Linear_forward = patches.undo(__name__, torch.nn.Linear, 'forward') # pylint: disable=E1128
|
||||
self.Linear_load_state_dict = patches.undo(__name__, torch.nn.Linear, '_load_from_state_dict') # pylint: disable=E1128
|
||||
|
||||
@@ -150,6 +150,10 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No
|
||||
recompile_model = True
|
||||
shared.compiled_model_state.lora_model = []
|
||||
break
|
||||
if not recompile_model:
|
||||
if len(loaded_networks) > 0 and debug:
|
||||
shared.log.debug('OpenVINO: Skipping LoRa loading')
|
||||
return
|
||||
else:
|
||||
recompile_model = True
|
||||
shared.compiled_model_state.lora_model = []
|
||||
@@ -166,11 +170,10 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No
|
||||
try:
|
||||
if recompile_model:
|
||||
shared.compiled_model_state.lora_model.append(f"{name}:{te_multipliers[i] if te_multipliers else 1.0}")
|
||||
if shared.backend == shared.Backend.DIFFUSERS and (os.environ.get('SD_LORA_DIFFUSERS', None)
|
||||
or getattr(network_on_disk, 'shorthash', None) == 'aaebf6360f7d' # lcm sd15
|
||||
or getattr(network_on_disk, 'shorthash', None) == '3d18b05e4f56' # lcm sdxl
|
||||
or (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx")):
|
||||
# OpenVINO only works with Diffusers LoRa loading.
|
||||
if shared.backend == shared.Backend.DIFFUSERS and (os.environ.get('SD_LORA_DIFFUSERS', None) is not None): # OpenVINO only works with Diffusers LoRa loading.
|
||||
# or getattr(network_on_disk, 'shorthash', '').lower() == 'aaebf6360f7d' # sd15-lcm
|
||||
# or getattr(network_on_disk, 'shorthash', '').lower() == '3d18b05e4f56' # sdxl-lcm
|
||||
# or getattr(network_on_disk, 'shorthash', '').lower() == '813ea5fb1c67' # turbo sdxl-turbo
|
||||
net = load_diffusers(name, network_on_disk, lora_scale=te_multipliers[i] if te_multipliers else 1.0)
|
||||
else:
|
||||
net = load_network(name, network_on_disk)
|
||||
|
||||
@@ -1,22 +1,16 @@
|
||||
import re
|
||||
import gradio as gr
|
||||
from fastapi import FastAPI
|
||||
import network
|
||||
import networks
|
||||
import lora # noqa:F401 # pylint: disable=unused-import
|
||||
# import lora_patches
|
||||
import extra_networks_lora
|
||||
import ui_extra_networks_lora
|
||||
from network import NetworkOnDisk
|
||||
from ui_extra_networks_lora import ExtraNetworksPageLora
|
||||
from extra_networks_lora import ExtraNetworkLora
|
||||
# import lora # noqa:F401 # pylint: disable=unused-import
|
||||
from modules import script_callbacks, ui_extra_networks, extra_networks, shared
|
||||
|
||||
|
||||
# def unload():
|
||||
# networks.originals.undo()
|
||||
|
||||
|
||||
def before_ui():
|
||||
ui_extra_networks.register_page(ui_extra_networks_lora.ExtraNetworksPageLora())
|
||||
networks.extra_network_lora = extra_networks_lora.ExtraNetworkLora()
|
||||
ui_extra_networks.register_page(ExtraNetworksPageLora())
|
||||
networks.extra_network_lora = ExtraNetworkLora()
|
||||
extra_networks.register_extra_network(networks.extra_network_lora)
|
||||
# extra_networks.register_extra_network_alias(networks.extra_network_lora, "lyco")
|
||||
|
||||
@@ -28,15 +22,7 @@ script_callbacks.on_before_ui(before_ui)
|
||||
script_callbacks.on_infotext_pasted(networks.infotext_pasted)
|
||||
|
||||
|
||||
shared.options_templates.update(shared.options_section(('extra_networks', "Extra Networks"), {
|
||||
# "sd_lora": shared.OptionInfo("None", "Add network to prompt", gr.Dropdown, lambda: {"choices": ["None", *networks.available_networks], "visible": False}, refresh=networks.list_available_networks),
|
||||
"sd_lora": shared.OptionInfo("None", "Add network to prompt", gr.Dropdown, {"choices": ["None"], "visible": False}),
|
||||
# "lora_show_all": shared.OptionInfo(False, "Always show all networks on the Lora page").info("otherwise, those detected as for incompatible version of Stable Diffusion will be hidden"),
|
||||
# "lora_hide_unknown_for_versions": shared.OptionInfo([], "Hide networks of unknown versions for model versions", gr.CheckboxGroup, {"choices": ["SD1", "SD2", "SDXL"]}),
|
||||
}))
|
||||
|
||||
|
||||
def create_lora_json(obj: network.NetworkOnDisk):
|
||||
def create_lora_json(obj: NetworkOnDisk):
|
||||
return {
|
||||
"name": obj.name,
|
||||
"alias": obj.alias,
|
||||
@@ -45,7 +31,7 @@ def create_lora_json(obj: network.NetworkOnDisk):
|
||||
}
|
||||
|
||||
|
||||
def api_networks(_: gr.Blocks, app: FastAPI):
|
||||
def api_networks(_, app: FastAPI):
|
||||
@app.get("/sdapi/v1/loras")
|
||||
async def get_loras():
|
||||
return [create_lora_json(obj) for obj in networks.available_networks.values()]
|
||||
|
||||
@@ -91,7 +91,7 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
|
||||
return None
|
||||
|
||||
def list_items(self):
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor:
|
||||
future_items = {executor.submit(self.create_item, net): net for net in networks.available_networks}
|
||||
for future in concurrent.futures.as_completed(future_items):
|
||||
item = future.result()
|
||||
|
||||
+2
-2
@@ -71,7 +71,7 @@
|
||||
"extra networks": [
|
||||
{"id":"","label":"UI position","localized":"","hint":"Location of extra networks"},
|
||||
{"id":"","label":"cover","localized":"","hint":"cover full area"},
|
||||
{"id":"","label":"inline","localized":"","hint":"inline with all additional elelemtns (scrollable)"},
|
||||
{"id":"","label":"inline","localized":"","hint":"inline with all additional elements (scrollable)"},
|
||||
{"id":"","label":"sidebar","localized":"","hint":"sidebar on the right side of the screen"},
|
||||
{"id":"","label":"UI height (%)","localized":"","hint":""},
|
||||
{"id":"","label":"UI sidebar width (%)","localized":"","hint":""},
|
||||
@@ -445,7 +445,7 @@
|
||||
{"id":"","label":"File format for grids","localized":"","hint":""},
|
||||
{"id":"","label":"Add extended info (seed, prompt) to filename when saving grid","localized":"","hint":""},
|
||||
{"id":"","label":"Grid row count","localized":"","hint":"Use -1 for autodetect and 0 for it to be same as batch size"},
|
||||
{"id":"","label":"Create text file next to every image with generation parameters","localized":"","hint":""},
|
||||
{"id":"","label":"Create info file for each every image","localized":"","hint":""},
|
||||
{"id":"","label":"Create JSON log file for each saved image","localized":"","hint":"Save image information to a JSON file"},
|
||||
{"id":"","label":"Save copy of image before doing face restoration","localized":"","hint":""},
|
||||
{"id":"","label":"Save copy of image before applying hires","localized":"","hint":""},
|
||||
|
||||
@@ -14,6 +14,26 @@
|
||||
"desc": "Stable Diffusion XL (SDXL) is the latest AI image generation model that is tailored towards more photorealistic outputs with more detailed imagery and composition compared to previous SD models, including SD 2.1. It can make realistic faces, legible text within the images, and better image composition, all while using shorter and simpler prompts at a greatly increased base resolution of 1024x1024. Just like its predecessors, SDXL has the ability to generate image variations using image-to-image prompting, inpainting (reimagining of the selected parts of an image), and outpainting (creating new parts that lie outside the image borders).",
|
||||
"preview": "stabilityai--stable-diffusion-xl-base-1.0.jpg"
|
||||
},
|
||||
"StabilityAI SD 2.1 Turbo": {
|
||||
"path": "stabilityai/sd-turbo",
|
||||
"desc": "SD-Turbo is a distilled version of Stable Diffusion 2.1, trained for real-time synthesis. SD-Turbo is based on a novel training method called Adversarial Diffusion Distillation (ADD) (see the technical report), which allows sampling large-scale foundational image diffusion models in 1 to 4 steps at high image quality. This approach uses score distillation to leverage large-scale off-the-shelf image diffusion models as a teacher signal and combines this with an adversarial loss to ensure high image fidelity even in the low-step regime of one or two sampling steps.",
|
||||
"preview": "stabilityai--sd-turbo.jpg"
|
||||
},
|
||||
"StabilityAI SD-XL Turbo": {
|
||||
"path": "stabilityai/sdxl-turbo",
|
||||
"desc": "SDXL-Turbo is a distilled version of SDXL 1.0, trained for real-time synthesis. SDXL-Turbo is based on a novel training method called Adversarial Diffusion Distillation (ADD) (see the technical report), which allows sampling large-scale foundational image diffusion models in 1 to 4 steps at high image quality. This approach uses score distillation to leverage large-scale off-the-shelf image diffusion models as a teacher signal and combines this with an adversarial loss to ensure high image fidelity even in the low-step regime of one or two sampling steps.",
|
||||
"preview": "stabilityai--sdxl-turbo.jpg"
|
||||
},
|
||||
"StabilityAI Stable Video Diffusion": {
|
||||
"path": "stabilityai/stable-video-diffusion-img2vid",
|
||||
"desc": "(SVD) Image-to-Video is a latent diffusion model trained to generate short video clips from an image conditioning. This model was trained to generate 14 frames at resolution 576x1024 given a context frame of the same size. We also finetune the widely used f8-decoder for temporal consistency.",
|
||||
"preview": "stabilityai--stable-video-diffusion-img2vid.jpg"
|
||||
},
|
||||
"StabilityAI Stable Video Diffusion XT": {
|
||||
"path": "stabilityai/stable-video-diffusion-img2vid-xt",
|
||||
"desc": "(SVD) Image-to-Video is a latent diffusion model trained to generate short video clips from an image conditioning. This model was trained to generate 25 frames at resolution 576x1024 given a context frame of the same size, finetuned from SVD Image-to-Video [14 frames]. We also finetune the widely used f8-decoder for temporal consistency.",
|
||||
"preview": "stabilityai--stable-video-diffusion-img2vid-xt.jpg"
|
||||
},
|
||||
"Segmind SSD-1B": {
|
||||
"path": "segmind/SSD-1B",
|
||||
"desc": "The Segmind Stable Diffusion Model (SSD-1B) offers a compact, efficient, and distilled version of the SDXL model. At 50% smaller and 60% faster than Stable Diffusion XL (SDXL), it provides quick and seamless performance without sacrificing image quality.",
|
||||
@@ -54,6 +74,11 @@
|
||||
"desc": "Kandinsky 2.2 is a text-conditional diffusion model (+0.1!) based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder. Kandinsky 2.1 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas. It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.",
|
||||
"preview": "kandinsky-community--kandinsky-2-2-decoder.jpg"
|
||||
},
|
||||
"Kandinsky 3": {
|
||||
"path": "kandinsky-community/kandinsky-3",
|
||||
"desc": "Kandinsky 3.0 is an open-source text-to-image diffusion model built upon the Kandinsky2-x model family. In comparison to its predecessors, Kandinsky 3.0 incorporates more data and specifically related to Russian culture, which allows to generate pictures related to Russin culture. Furthermore, enhancements have been made to the text understanding and visual quality of the model, achieved by increasing the size of the text encoder and Diffusion U-Net models, respectively.",
|
||||
"preview": "kandinsky-community--kandinsky-3.jpg"
|
||||
},
|
||||
"DeepFloyd IF Medium": {
|
||||
"path": "DeepFloyd/IF-I-M-v1.0",
|
||||
"desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.",
|
||||
|
||||
+16
-25
@@ -6,8 +6,6 @@ import shutil
|
||||
import logging
|
||||
import platform
|
||||
import subprocess
|
||||
import io
|
||||
import pstats
|
||||
import cProfile
|
||||
import pkg_resources
|
||||
|
||||
@@ -29,6 +27,7 @@ opts = {}
|
||||
args = Dot({
|
||||
'debug': False,
|
||||
'reset': False,
|
||||
'profile': False,
|
||||
'upgrade': False,
|
||||
'skip_extensions': False,
|
||||
'skip_requirements': False,
|
||||
@@ -103,12 +102,6 @@ def setup_logging():
|
||||
fh.doRollover()
|
||||
log_rolled = True
|
||||
|
||||
global first_call # pylint: disable=global-statement
|
||||
if first_call:
|
||||
log_size = os.path.getsize(log_file) if os.path.exists(log_file) else 0
|
||||
log.debug(f'Logger: file={log_file} level={level} size={log_size} mode={"append" if not log_rolled else "create"}')
|
||||
first_call = False
|
||||
|
||||
fh.formatter = logging.Formatter('%(asctime)s | %(name)s | %(levelname)s | %(module)s | %(message)s')
|
||||
fh.setLevel(logging.DEBUG)
|
||||
log.addHandler(fh)
|
||||
@@ -121,10 +114,17 @@ def setup_logging():
|
||||
# overrides
|
||||
logging.getLogger("urllib3").setLevel(logging.ERROR)
|
||||
logging.getLogger("httpx").setLevel(logging.ERROR)
|
||||
logging.getLogger("diffusers").setLevel(logging.ERROR)
|
||||
logging.getLogger("torch").setLevel(logging.ERROR)
|
||||
logging.getLogger("ControlNet").handlers = log.handlers
|
||||
logging.getLogger("lycoris").handlers = log.handlers
|
||||
# logging.getLogger("DeepSpeed").handlers = log.handlers
|
||||
|
||||
def get_logfile():
|
||||
log_size = os.path.getsize(log_file) if os.path.exists(log_file) else 0
|
||||
log.info(f'Logger: file="{log_file}" level={logging.getLevelName(logging.DEBUG if args.debug else logging.INFO)} size={log_size} mode={"append" if not log_rolled else "create"}')
|
||||
return log_file
|
||||
|
||||
|
||||
def custom_excepthook(exc_type, exc_value, exc_traceback):
|
||||
import traceback
|
||||
@@ -142,19 +142,9 @@ def print_dict(d):
|
||||
return ' '.join([f'{k}={v}' for k, v in d.items()])
|
||||
|
||||
|
||||
def print_profile(profile: cProfile.Profile, msg: str):
|
||||
try:
|
||||
from rich import print # pylint: disable=redefined-builtin
|
||||
except Exception:
|
||||
pass
|
||||
profile.disable()
|
||||
stream = io.StringIO()
|
||||
ps = pstats.Stats(profile, stream=stream)
|
||||
ps.sort_stats(pstats.SortKey.CUMULATIVE).print_stats(15)
|
||||
profile = None
|
||||
lines = stream.getvalue().split('\n')
|
||||
lines = [line for line in lines if '<frozen' not in line and '{built-in' not in line and '/logging' not in line and '/rich' not in line]
|
||||
print(f'Profile {msg}:', '\n'.join(lines))
|
||||
def print_profile(profiler: cProfile.Profile, msg: str):
|
||||
from modules.errors import profile
|
||||
profile(profiler, msg)
|
||||
|
||||
|
||||
# check if package is installed
|
||||
@@ -421,7 +411,6 @@ def check_torch():
|
||||
log.debug(f'ROCm hipconfig failed: {e}')
|
||||
rocm_ver = None
|
||||
if rocm_ver in {"5.7"}:
|
||||
# install torch nightly via torchvision to avoid wasting bandwidth when torchvision depends on torch from yesterday
|
||||
torch_command = os.environ.get('TORCH_COMMAND', f'torch torchvision --pre --index-url https://download.pytorch.org/whl/nightly/rocm{rocm_ver}')
|
||||
elif rocm_ver in {"5.5", "5.6"}:
|
||||
torch_command = os.environ.get('TORCH_COMMAND', f'torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm{rocm_ver}')
|
||||
@@ -443,7 +432,7 @@ def check_torch():
|
||||
ipex_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.0.110%2Bxpu-master%2Bdll-bundle/intel_extension_for_pytorch-2.0.110+gitc6ea20b-cp310-cp310-win_amd64.whl'
|
||||
torch_command = os.environ.get('TORCH_COMMAND', f'{pytorch_pip} {torchvision_pip} {ipex_pip}')
|
||||
install('openvino', 'openvino', ignore=True)
|
||||
install('onnxruntime-openvino', 'onnxruntime-openvino', ignore=True) # TODO numpy version conflicts with tensorflow and doesn't support Python 3.11
|
||||
install('onnxruntime-openvino', 'onnxruntime-openvino', ignore=True)
|
||||
elif allow_openvino and args.use_openvino:
|
||||
log.info('Using OpenVINO')
|
||||
torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.1.1 torchvision==0.16.1 --index-url https://download.pytorch.org/whl/cpu')
|
||||
@@ -514,10 +503,11 @@ def check_torch():
|
||||
if opts.get('cuda_compile_backend', '') == 'hidet':
|
||||
install('hidet', 'hidet')
|
||||
if args.use_openvino or opts.get('cuda_compile_backend', '') == 'openvino_fx':
|
||||
uninstall('openvino-nightly') # TODO remove after people had enough time upgrading
|
||||
uninstall('openvino-nightly') # TODO openvino: remove after people had enough time upgrading
|
||||
install('openvino==2023.2.0', 'openvino')
|
||||
install('onnxruntime-openvino', 'onnxruntime-openvino', ignore=True) # TODO numpy version conflicts with tensorflow and doesn't support Python 3.11
|
||||
install('onnxruntime-openvino', 'onnxruntime-openvino', ignore=True) # TODO openvino: numpy version conflicts with tensorflow and doesn't support Python 3.11
|
||||
os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX')
|
||||
os.environ.setdefault('SD_LORA_DIFFUSERS', '1')
|
||||
os.environ.setdefault('NEOReadDebugKeys', '1')
|
||||
os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100')
|
||||
if args.profile:
|
||||
@@ -767,6 +757,7 @@ def set_environment():
|
||||
os.environ.setdefault('TF_ENABLE_ONEDNN_OPTS', '0')
|
||||
os.environ.setdefault('USE_TORCH', '1')
|
||||
os.environ.setdefault('UVICORN_TIMEOUT_KEEP_ALIVE', '60')
|
||||
os.environ.setdefault('KINETO_LOG_LEVEL', '3')
|
||||
os.environ.setdefault('HF_HUB_CACHE', opts.get('hfcache_dir', os.path.join(os.path.expanduser('~'), '.cache', 'huggingface', 'hub')))
|
||||
log.debug(f'Cache folder: {os.environ.get("HF_HUB_CACHE")}')
|
||||
if sys.platform == 'darwin':
|
||||
|
||||
@@ -166,8 +166,8 @@ function sortExtraNetworks() {
|
||||
if (num === 0) return 'sort: no cards';
|
||||
cards.sort((a, b) => { // eslint-disable-line no-loop-func
|
||||
switch (sortVal) {
|
||||
case 0: return a.dataset.name ? a.dataset.search.localeCompare(b.dataset.name) : 0;
|
||||
case 1: return b.dataset.name ? b.dataset.search.localeCompare(a.dataset.name) : 0;
|
||||
case 0: return a.dataset.search ? a.dataset.search.localeCompare(b.dataset.search) : 0;
|
||||
case 1: return b.dataset.search ? b.dataset.search.localeCompare(a.dataset.search) : 0;
|
||||
case 2: return a.dataset.mtime && !isNaN(a.dataset.mtime) ? parseFloat(b.dataset.mtime) - parseFloat(a.dataset.mtime) : 0;
|
||||
case 3: return b.dataset.mtime && !isNaN(b.dataset.mtime) ? parseFloat(a.dataset.mtime) - parseFloat(b.dataset.mtime) : 0;
|
||||
case 4: return a.dataset.size && !isNaN(a.dataset.size) ? parseFloat(b.dataset.size) - parseFloat(a.dataset.size) : 0;
|
||||
|
||||
+13
-13
@@ -95,7 +95,7 @@ div#extras_scale_to_tab div.form{ flex-direction: row; }
|
||||
#img2img_sketch, #img2maskimg, #inpaint_sketch { overflow: overlay !important; resize: auto; background: var(--panel-background-fill); z-index: 5; }
|
||||
.image-buttons button{ min-width: auto; }
|
||||
.infotext { overflow-wrap: break-word; line-height: 1.5em; }
|
||||
.infotext > p { padding-left: 1em; text-indent: -1em; }
|
||||
.infotext > p { padding-left: 1em; text-indent: -1em; white-space: pre-wrap; }
|
||||
.tooltip { display: block; position: fixed; top: 1em; right: 1em; padding: 0.5em; background: var(--input-background-fill); color: var(--body-text-color); border: 1pt solid var(--button-primary-border-color);
|
||||
width: 22em; min-height: 1.3em; font-size: 0.8em; transition: opacity 0.2s ease-in; pointer-events: none; opacity: 0; z-index: 999; }
|
||||
.tooltip-show { opacity: 0.9; }
|
||||
@@ -184,7 +184,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
|
||||
.extra-networks .description { flex: 3; }
|
||||
.extra-networks .tab-nav > button { margin-right: 0; height: 24px; padding: 2px 4px 2px 4px; }
|
||||
.extra-networks .buttons { position: absolute; right: 0; margin: -4px; background: var(--background-color); }
|
||||
.extra-networks .buttons > button { margin-left: -0.4em; height: 1.4em; color: var(--primary-300) !important; }
|
||||
.extra-networks .buttons > button { margin-left: -0.2em; height: 1.4em; color: var(--primary-300) !important; }
|
||||
.extra-networks .custom-button { width: 120px; width: 100%; background: none; justify-content: left; text-align: left; padding: 3px 3px 3px 12px; text-indent: -6px; box-shadow: none; line-break: auto; }
|
||||
.extra-networks .custom-button:hover { background: var(--button-primary-background-fill) }
|
||||
.extra-networks-tab { padding: 0 !important; }
|
||||
@@ -286,25 +286,25 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
|
||||
/* Do not affect displays larger than 1024px wide. */
|
||||
@media (max-width: 1024px) {
|
||||
|
||||
/* Screens smaller than 424px wide */
|
||||
/* Screens smaller than 400px wide */
|
||||
@media (max-width: 399px) {
|
||||
:root, .light, .dark { --left-column: 100%; }
|
||||
|
||||
/* maintain single column for from image operations on larger mobile devices */
|
||||
#txt2img_results, #img2img_results, #extras_results { min-width: calc(min(320px, 100%)) !important;}
|
||||
#txt2img_footer p { text-wrap: wrap; }
|
||||
|
||||
}
|
||||
|
||||
/* Screens larger than 425px wide */
|
||||
@media (min-width: 425px) {
|
||||
:root, .light, .dark {--left-column: 50% ;}
|
||||
|
||||
/* adjust extension panel to fit within resized sidebar */
|
||||
#scripts_alwayson_txt2img div { max-width: 99%; }
|
||||
/* Screens larger than 400px wide */
|
||||
@media (min-width: 400px) {
|
||||
:root, .light, .dark {--left-column: 50%;}
|
||||
|
||||
/* maintain side by side split on larger mobile displays for from text */
|
||||
#txt2img_results, #extras_results { min-width: 50% !important;}
|
||||
#txt2img_results, #extras_results, #txt2img_footer p {text-wrap: wrap; max-width: 100% !important; }
|
||||
}
|
||||
|
||||
#scripts_alwayson_txt2img div, #scripts_alwayson_img2img div { max-width: 100%; }
|
||||
#txt2img_prompt_container, #img2img_prompt_container { resize:vertical !important; }
|
||||
|
||||
/* make generate and enqueue buttons take up the entire width of their rows. */
|
||||
@@ -316,10 +316,10 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
|
||||
#txt2img_generate_box, #img2img_generate_box, #txt2img_enqueue_wrapper,#img2img_enqueue_wrapper {display: flex;flex-direction: column;height: 4em !important;align-items: stretch;justify-content: space-evenly;}
|
||||
|
||||
/* maintain single column for from image operations on larger mobile devices */
|
||||
#img2img_settings, #img2img_results { min-width: 100% !important; max-width: 100% !important;}
|
||||
#img2img_interface, #img2img_results, #img2img_footer p {text-wrap: wrap; min-width: 100% !important; max-width: 100% !important;}
|
||||
/* fix inpaint image display being too large for mobile displays */
|
||||
#img2img_sketch, #img2maskimg, #inpaint_sketch {display: flex;alignment-baseline:after-edge !important;overflow: auto !important;resize: none !important;}
|
||||
#img2maskimg canvas { width: 100% !important; max-height: 100% !important; height: auto !important; }
|
||||
#img2img_sketch, #img2maskimg, #inpaint_sketch {display: flex; alignment-baseline:after-edge !important; overflow: auto !important; resize: none !important; }
|
||||
#img2maskimg canvas { width: auto !important; max-height: 100% !important; height: auto !important; }
|
||||
|
||||
/* fix from text/image UI elements to prevent them from moving around within the UI */
|
||||
#txt2img_sampler, #txt2img_batch, #txt2img_seed_group, #txt2img_advanced, #txt2img_second_pass, #img2img_sampling_group, #img2img_resize_group, #img2img_batch_group, #img2img_seed_group, #img2img_denoise_group, #img2img_advanced_group { width: 100% !important; }
|
||||
|
||||
@@ -4,7 +4,6 @@ import os
|
||||
import sys
|
||||
import time
|
||||
import shlex
|
||||
import logging
|
||||
import subprocess
|
||||
from functools import lru_cache
|
||||
import installer
|
||||
@@ -171,6 +170,7 @@ if __name__ == "__main__":
|
||||
installer.args = args
|
||||
installer.setup_logging()
|
||||
installer.log.info('Starting SD.Next')
|
||||
installer.get_logfile()
|
||||
try:
|
||||
sys.excepthook = installer.custom_excepthook
|
||||
except Exception:
|
||||
@@ -218,9 +218,6 @@ if __name__ == "__main__":
|
||||
installer.log.warning(f'See log file for more details: {installer.log_file}')
|
||||
installer.extensions_preload(parser) # adds additional args from extensions
|
||||
args = installer.parse_args(parser)
|
||||
# installer.run_setup()
|
||||
# installer.log.debug(f"Args: {vars(args)}")
|
||||
logging.disable(logging.NOTSET if args.debug else logging.DEBUG)
|
||||
|
||||
uv, instance = start_server(immediate=True, server=None)
|
||||
while True:
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 342 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 342 KiB |
@@ -2,9 +2,6 @@ import html
|
||||
import threading
|
||||
import time
|
||||
import cProfile
|
||||
import pstats
|
||||
import io
|
||||
from rich import print # pylint: disable=redefined-builtin
|
||||
from modules import shared, progress, errors
|
||||
|
||||
queue_lock = threading.Lock()
|
||||
@@ -62,10 +59,7 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False, name=None):
|
||||
else:
|
||||
res = list(res)
|
||||
if shared.cmd_opts.profile:
|
||||
pr.disable()
|
||||
s = io.StringIO()
|
||||
pstats.Stats(pr, stream=s).sort_stats(pstats.SortKey.CUMULATIVE).print_stats(15)
|
||||
print('Profile Exec:', s.getvalue())
|
||||
errors.profile(pr, 'Wrap')
|
||||
except Exception as e:
|
||||
errors.display(e, 'gradio call')
|
||||
if extra_outputs_array is None:
|
||||
|
||||
@@ -150,6 +150,25 @@ def torch_gc(force=False):
|
||||
log.debug(f'gc: collected={collected} device={torch.device(get_optimal_device_name())} {memstats.memory_stats()}')
|
||||
|
||||
|
||||
def set_cuda_sync_mode(mode):
|
||||
"""
|
||||
Set the CUDA device synchronization mode: auto, spin, yield or block.
|
||||
auto: Chooses spin or yield depending on the number of available CPU cores.
|
||||
spin: Runs one CPU core per GPU at 100% to poll for completed operations.
|
||||
yield: Gives control to other threads between polling, if any are waiting.
|
||||
block: Lets the thread sleep until the GPU driver signals completion.
|
||||
"""
|
||||
if mode == -1 or mode == 'none' or not cuda_ok:
|
||||
return
|
||||
try:
|
||||
import ctypes
|
||||
log.info(f'Set cuda synch: mode={mode}')
|
||||
torch.cuda.set_device(torch.device(get_optimal_device_name()))
|
||||
ctypes.CDLL('libcudart.so').cudaSetDeviceFlags({'auto': 0, 'spin': 1, 'yield': 2, 'block': 4}[mode])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def test_fp16():
|
||||
if shared.cmd_opts.experimental:
|
||||
return True
|
||||
@@ -272,6 +291,9 @@ dtype = torch.float16
|
||||
dtype_vae = torch.float16
|
||||
dtype_unet = torch.float16
|
||||
unet_needs_upcast = False
|
||||
if args.profile:
|
||||
log.info(f'Torch build config: {torch.__config__.show()}')
|
||||
# set_cuda_sync_mode('block') # none/auto/spin/yield/block
|
||||
|
||||
|
||||
def cond_cast_unet(tensor):
|
||||
|
||||
@@ -55,3 +55,41 @@ def run(code, task):
|
||||
|
||||
def exception(suppress=[]): # noqa: B006
|
||||
console.print_exception(show_locals=False, max_frames=10, extra_lines=2, suppress=suppress, theme="ansi_dark", word_wrap=False, width=min([console.width, 200]))
|
||||
|
||||
|
||||
def profile(profiler, msg: str):
|
||||
profiler.disable()
|
||||
import io
|
||||
import pstats
|
||||
stream = io.StringIO() # pylint: disable=abstract-class-instantiated
|
||||
p = pstats.Stats(profiler, stream=stream)
|
||||
p.sort_stats(pstats.SortKey.CUMULATIVE)
|
||||
p.print_stats(100)
|
||||
# p.print_title()
|
||||
# p.print_call_heading(10, 'time')
|
||||
# p.print_callees(10)
|
||||
# p.print_callers(10)
|
||||
profiler = None
|
||||
lines = stream.getvalue().split('\n')
|
||||
lines = [x for x in lines if '<frozen' not in x
|
||||
and '{built-in' not in x
|
||||
and '/logging' not in x
|
||||
and 'Ordered by' not in x
|
||||
and 'List reduced' not in x
|
||||
and '_lsprof' not in x
|
||||
and '/profiler' not in x
|
||||
and 'rich' not in x
|
||||
and x.strip() != ''
|
||||
]
|
||||
txt = '\n'.join(lines[:min(5, len(lines))])
|
||||
log.debug(f'Profile {msg}: {txt}')
|
||||
|
||||
|
||||
def profile_torch(profiler, msg: str):
|
||||
profiler.stop()
|
||||
lines = profiler.key_averages().table(sort_by="self_cpu_time_total", row_limit=12)
|
||||
lines = lines.split('\n')
|
||||
lines = [x for x in lines if '/profiler' not in x and '---' not in x]
|
||||
txt = '\n'.join(lines)
|
||||
# print(f'Torch {msg}:', txt)
|
||||
log.debug(f'Torch profile {msg}: \n{txt}')
|
||||
|
||||
+1
-1
@@ -134,7 +134,7 @@ def run_modelmerger(id_task, **kwargs): # pylint: disable=unused-argument
|
||||
|
||||
try:
|
||||
theta_0 = theta_0.to_dict() #TensorDict -> Dict if necessary
|
||||
except:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
bake_in_vae_filename = sd_vae.vae_dict.get(kwargs.get("bake_in_vae", None), None)
|
||||
|
||||
+71
-12
@@ -244,6 +244,8 @@ def resize_image(resize_mode, im, width, height, upscaler_name=None, output_type
|
||||
|
||||
if resize_mode == 0:
|
||||
res = im.copy()
|
||||
if width == 0 or height == 0:
|
||||
res = im.copy()
|
||||
elif resize_mode == 1:
|
||||
res = resize(im, width, height)
|
||||
elif resize_mode == 2:
|
||||
@@ -425,6 +427,20 @@ class FilenameGenerator:
|
||||
debug(f'Filename sanitize: input="{filename}" parts={parts} output="{fn}" ext={ext} max={max_length} len={len(fn)}')
|
||||
return fn
|
||||
|
||||
def sequence(self, x, dirname, basename):
|
||||
if shared.opts.save_images_add_number or '[seq]' in x:
|
||||
if '[seq]' not in x:
|
||||
x = os.path.join(os.path.dirname(x), f"[seq]-{os.path.basename(x)}")
|
||||
basecount = get_next_sequence_number(dirname, basename)
|
||||
for i in range(9999):
|
||||
seq = f"{basecount + i:05}" if basename == '' else f"{basename}-{basecount + i:04}"
|
||||
filename = x.replace('[seq]', seq)
|
||||
if not os.path.exists(filename):
|
||||
debug(f'Prompt sequence: input="{x}" seq={seq} output="{filename}"')
|
||||
x = filename
|
||||
break
|
||||
return x
|
||||
|
||||
def apply(self, x):
|
||||
res = ''
|
||||
for m in re_pattern.finditer(x):
|
||||
@@ -591,18 +607,7 @@ def save_image(image, path, basename='', seed=None, prompt=None, extension=share
|
||||
dirname = os.path.dirname(params.filename)
|
||||
if dirname is not None and len(dirname) > 0:
|
||||
os.makedirs(dirname, exist_ok=True)
|
||||
# sequence
|
||||
if shared.opts.save_images_add_number or '[seq]' in params.filename:
|
||||
if '[seq]' not in params.filename:
|
||||
params.filename = os.path.join(os.path.dirname(params.filename), f"[seq]-{os.path.basename(params.filename)}")
|
||||
basecount = get_next_sequence_number(dirname, basename)
|
||||
for i in range(9999):
|
||||
seq = f"{basecount + i:05}" if basename == '' else f"{basename}-{basecount + i:04}"
|
||||
filename = params.filename.replace('[seq]', seq)
|
||||
if not os.path.exists(filename):
|
||||
debug(f'Prompt sequence: input="{params.filename}" seq={seq} output="{filename}"')
|
||||
params.filename = filename
|
||||
break
|
||||
params.filename = namegen.sequence(params.filename, dirname, basename)
|
||||
# callbacks
|
||||
script_callbacks.before_image_saved_callback(params)
|
||||
exifinfo = params.pnginfo.get('UserComment', '')
|
||||
@@ -618,6 +623,60 @@ def save_image(image, path, basename='', seed=None, prompt=None, extension=share
|
||||
return params.filename, filename_txt
|
||||
|
||||
|
||||
def save_video_atomic(images, filename, video_type: str = 'none', duration: float = 2.0, loop: bool = False, interpolate: int = 0, scale: float = 1.0, pad: int = 1, change: float = 0.3):
|
||||
try:
|
||||
import cv2
|
||||
except Exception as e:
|
||||
shared.log.error(f'Save video: cv2: {e}')
|
||||
return
|
||||
os.makedirs(os.path.dirname(filename), exist_ok=True)
|
||||
if video_type.lower() == 'mp4':
|
||||
frames = images
|
||||
if interpolate > 0:
|
||||
try:
|
||||
import modules.rife
|
||||
frames = modules.rife.interpolate(images, count=interpolate, scale=scale, pad=pad, change=change)
|
||||
except Exception as e:
|
||||
shared.log.error(f'RIFE interpolation: {e}')
|
||||
errors.display(e, 'RIFE interpolation')
|
||||
video_frames = [np.array(frame) for frame in frames]
|
||||
fourcc = "mp4v"
|
||||
h, w, _c = video_frames[0].shape
|
||||
video_writer = cv2.VideoWriter(filename, fourcc=cv2.VideoWriter_fourcc(*fourcc), fps=len(frames)/duration, frameSize=(w, h))
|
||||
for i in range(len(video_frames)):
|
||||
img = cv2.cvtColor(video_frames[i], cv2.COLOR_RGB2BGR)
|
||||
video_writer.write(img)
|
||||
shared.log.info(f'Save video: file="{filename}" frames={len(frames)} duration={duration} fourcc={fourcc}')
|
||||
if video_type.lower() == 'gif' or video_type.lower() == 'png':
|
||||
append = images.copy()
|
||||
image = append.pop(0)
|
||||
if loop:
|
||||
append += append[::-1]
|
||||
frames=len(append) + 1
|
||||
image.save(
|
||||
filename,
|
||||
save_all = True,
|
||||
append_images = append,
|
||||
optimize = False,
|
||||
duration = 1000.0 * duration / frames,
|
||||
loop = 0 if loop else 1,
|
||||
)
|
||||
shared.log.info(f'Save video: file="{filename}" frames={len(append) + 1} duration={duration} loop={loop}')
|
||||
|
||||
|
||||
def save_video(p, images, filename = None, video_type: str = 'none', duration: float = 2.0, loop: bool = False, interpolate: int = 0, scale: float = 1.0, pad: int = 1, change: float = 0.3):
|
||||
if images is None or len(images) < 2 or video_type is None or video_type.lower() == 'none':
|
||||
return
|
||||
image = images[0]
|
||||
namegen = FilenameGenerator(p, seed=p.all_seeds[0], prompt=p.all_prompts[0], image=image)
|
||||
if filename is None:
|
||||
filename = namegen.apply(shared.opts.samples_filename_pattern if shared.opts.samples_filename_pattern and len(shared.opts.samples_filename_pattern) > 0 else "[seq]-[prompt_words]")
|
||||
filename = namegen.sanitize(os.path.join(shared.opts.outdir_video, filename))
|
||||
filename = namegen.sequence(filename, shared.opts.outdir_video, '')
|
||||
filename = f'{filename}.{video_type.lower()}'
|
||||
threading.Thread(target=save_video_atomic, args=(images, filename, video_type, duration, loop, interpolate, scale, pad, change)).start()
|
||||
|
||||
|
||||
def safe_decode_string(s: bytes):
|
||||
remove_prefix = lambda text, prefix: text[len(prefix):] if text.startswith(prefix) else text # pylint: disable=unnecessary-lambda-assignment
|
||||
for encoding in ['utf-8', 'utf-16', 'ascii', 'latin_1', 'cp1252', 'cp437']: # try different encodings
|
||||
|
||||
@@ -208,7 +208,6 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s
|
||||
p.scale_by = scale_by
|
||||
p.scripts = modules.scripts.scripts_img2img
|
||||
p.script_args = args
|
||||
p.extra_generation_params['Resize mode'] = resize_mode
|
||||
if mask:
|
||||
p.extra_generation_params["Mask blur"] = mask_blur
|
||||
p.extra_generation_params["Mask alpha"] = mask_alpha
|
||||
|
||||
@@ -31,6 +31,7 @@ def ipex_init(): # pylint: disable=too-many-statements
|
||||
torch.cuda.FloatTensor = torch.xpu.FloatTensor
|
||||
torch.Tensor.cuda = torch.Tensor.xpu
|
||||
torch.Tensor.is_cuda = torch.Tensor.is_xpu
|
||||
torch.UntypedStorage.cuda = torch.UntypedStorage.xpu
|
||||
torch.cuda._initialization_lock = torch.xpu.lazy_init._initialization_lock
|
||||
torch.cuda._initialized = torch.xpu.lazy_init._initialized
|
||||
torch.cuda._lazy_seed_tracker = torch.xpu.lazy_init._lazy_seed_tracker
|
||||
|
||||
@@ -93,13 +93,25 @@ def linalg_solve(A, B, *args, **kwargs): # pylint: disable=invalid-name
|
||||
else:
|
||||
return original_linalg_solve(A, B, *args, **kwargs)
|
||||
|
||||
def is_cuda(self):
|
||||
return self.device.type == 'xpu'
|
||||
|
||||
def ipex_hijacks():
|
||||
CondFunc('torch.tensor',
|
||||
lambda orig_func, *args, device=None, **kwargs: orig_func(*args, device=return_xpu(device), **kwargs),
|
||||
lambda orig_func, *args, device=None, **kwargs: check_device(device))
|
||||
CondFunc('torch.Tensor.to',
|
||||
lambda orig_func, self, device=None, *args, **kwargs: orig_func(self, return_xpu(device), *args, **kwargs),
|
||||
lambda orig_func, self, device=None, *args, **kwargs: check_device(device))
|
||||
CondFunc('torch.Tensor.cuda',
|
||||
lambda orig_func, self, device=None, *args, **kwargs: orig_func(self, return_xpu(device), *args, **kwargs),
|
||||
lambda orig_func, self, device=None, *args, **kwargs: check_device(device))
|
||||
CondFunc('torch.UntypedStorage.__init__',
|
||||
lambda orig_func, *args, device=None, **kwargs: orig_func(*args, device=return_xpu(device), **kwargs),
|
||||
lambda orig_func, *args, device=None, **kwargs: check_device(device))
|
||||
CondFunc('torch.UntypedStorage.cuda',
|
||||
lambda orig_func, self, device=None, *args, **kwargs: orig_func(self, return_xpu(device), *args, **kwargs),
|
||||
lambda orig_func, self, device=None, *args, **kwargs: check_device(device))
|
||||
CondFunc('torch.empty',
|
||||
lambda orig_func, *args, device=None, **kwargs: orig_func(*args, device=return_xpu(device), **kwargs),
|
||||
lambda orig_func, *args, device=None, **kwargs: check_device(device))
|
||||
@@ -112,9 +124,6 @@ def ipex_hijacks():
|
||||
CondFunc('torch.zeros',
|
||||
lambda orig_func, *args, device=None, **kwargs: orig_func(*args, device=return_xpu(device), **kwargs),
|
||||
lambda orig_func, *args, device=None, **kwargs: check_device(device))
|
||||
CondFunc('torch.tensor',
|
||||
lambda orig_func, *args, device=None, **kwargs: orig_func(*args, device=return_xpu(device), **kwargs),
|
||||
lambda orig_func, *args, device=None, **kwargs: check_device(device))
|
||||
CondFunc('torch.linspace',
|
||||
lambda orig_func, *args, device=None, **kwargs: orig_func(*args, device=return_xpu(device), **kwargs),
|
||||
lambda orig_func, *args, device=None, **kwargs: check_device(device))
|
||||
@@ -124,7 +133,7 @@ def ipex_hijacks():
|
||||
lambda orig_func, f, map_location=None, pickle_module=None, *, weights_only=False, mmap=None, **kwargs: check_device(map_location))
|
||||
|
||||
CondFunc('torch.Generator',
|
||||
lambda orig_func, device=None: torch.xpu.Generator(device),
|
||||
lambda orig_func, device=None: torch.xpu.Generator(return_xpu(device)),
|
||||
lambda orig_func, device=None: device is not None and device != torch.device("cpu") and device != "cpu")
|
||||
|
||||
#TiledVAE and ControlNet:
|
||||
@@ -180,5 +189,6 @@ def ipex_hijacks():
|
||||
torch.autocast = ipex_autocast
|
||||
torch.cat = torch_cat
|
||||
torch.linalg.solve = linalg_solve
|
||||
torch.UntypedStorage.is_cuda = is_cuda
|
||||
torch.nn.functional.interpolate = interpolate
|
||||
torch.backends.cuda.sdp_kernel = return_null_context
|
||||
|
||||
@@ -42,3 +42,16 @@ errors.install([gradio])
|
||||
import diffusers # pylint: disable=W0611,C0411
|
||||
timer.startup.record("diffusers")
|
||||
errors.log.info(f'Load packages: torch={getattr(torch, "__long_version__", torch.__version__)} diffusers={diffusers.__version__} gradio={gradio.__version__}')
|
||||
|
||||
try:
|
||||
import os
|
||||
import math
|
||||
cores = os.cpu_count()
|
||||
affinity = len(os.sched_getaffinity(0))
|
||||
threads = torch.get_num_threads()
|
||||
if threads < (affinity / 2):
|
||||
torch.set_num_threads(math.floor(affinity / 2))
|
||||
threads = torch.get_num_threads()
|
||||
errors.log.debug(f'Detected: cores={cores} affinity={affinity} set threads={threads}')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
+1
-1
Submodule modules/lora updated: 95ae56bd22...0908c5414d
@@ -13,10 +13,11 @@ from modules.merging import merge_methods
|
||||
from modules.merging.merge_utils import WeightClass
|
||||
from modules.merging.merge_rebasin import (
|
||||
apply_permutation,
|
||||
sdunet_permutation_spec,
|
||||
update_model_a,
|
||||
weight_matching,
|
||||
)
|
||||
from modules.merging.merge_PermSpec import sdunet_permutation_spec
|
||||
from modules.merging.merge_PermSpec_SDXL import sdxl_permutation_spec
|
||||
##########################################################
|
||||
# Files in modules.merging are heavily modified
|
||||
# versions of sd-meh by @s1dxl used with his blessing
|
||||
@@ -239,7 +240,10 @@ def rebasin_merge(
|
||||
):
|
||||
# not sure how this does when 3 models are involved...
|
||||
model_a = thetas["model_a"].clone()
|
||||
perm_spec = sdunet_permutation_spec()
|
||||
if weight_matcher.SDXL:
|
||||
perm_spec = sdxl_permutation_spec()
|
||||
else:
|
||||
perm_spec = sdunet_permutation_spec()
|
||||
|
||||
for it in range(iterations):
|
||||
log_vram(f"rebasin: iteration={it}")
|
||||
@@ -319,7 +323,7 @@ def merge_key( # pylint: disable=inconsistent-return-statements
|
||||
|
||||
for theta in thetas.values():
|
||||
if key not in theta.keys():
|
||||
return
|
||||
return thetas["model_a"][key]
|
||||
|
||||
current_bases = weight_matcher(key)
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
from modules.merging.merge_rebasin import PermutationSpec, permutation_spec_from_axes_to_perm
|
||||
def sdunet_permutation_spec() -> PermutationSpec:
|
||||
conv = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment
|
||||
f"{name}.weight": (
|
||||
p_out,
|
||||
p_in,
|
||||
),
|
||||
f"{name}.bias": (p_out,),
|
||||
}
|
||||
norm = lambda name, p: {f"{name}.weight": (p,), f"{name}.bias": (p,)} # pylint: disable=unnecessary-lambda-assignment
|
||||
dense = (
|
||||
lambda name, p_in, p_out, bias=True: { # pylint: disable=unnecessary-lambda-assignment
|
||||
f"{name}.weight": (p_out, p_in),
|
||||
f"{name}.bias": (p_out,),
|
||||
}
|
||||
if bias
|
||||
else {f"{name}.weight": (p_out, p_in)}
|
||||
)
|
||||
skip = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment
|
||||
f"{name}": (
|
||||
p_out,
|
||||
p_in,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
# Unet Res blocks
|
||||
easyblock = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment
|
||||
**norm(f"{name}.in_layers.0", p_in),
|
||||
**conv(f"{name}.in_layers.2", p_in, f"P_{name}_inner"),
|
||||
**dense(
|
||||
f"{name}.emb_layers.1", f"P_{name}_inner2", f"P_{name}_inner3", bias=True
|
||||
),
|
||||
**norm(f"{name}.out_layers.0", f"P_{name}_inner4"),
|
||||
**conv(f"{name}.out_layers.3", f"P_{name}_inner4", p_out),
|
||||
}
|
||||
|
||||
# VAE blocks - Unused
|
||||
easyblock2 = lambda name, p: { # pylint: disable=unnecessary-lambda-assignment, unused-variable # noqa: F841
|
||||
**norm(f"{name}.norm1", p),
|
||||
**conv(f"{name}.conv1", p, f"P_{name}_inner"),
|
||||
**norm(f"{name}.norm2", f"P_{name}_inner"),
|
||||
**conv(f"{name}.conv2", f"P_{name}_inner", p),
|
||||
}
|
||||
|
||||
# This is for blocks that use a residual connection, but change the number of channels via a Conv.
|
||||
shortcutblock = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment, , unused-variable # noqa: F841
|
||||
**norm(f"{name}.norm1", p_in),
|
||||
**conv(f"{name}.conv1", p_in, f"P_{name}_inner"),
|
||||
**norm(f"{name}.norm2", f"P_{name}_inner"),
|
||||
**conv(f"{name}.conv2", f"P_{name}_inner", p_out),
|
||||
**conv(f"{name}.nin_shortcut", p_in, p_out),
|
||||
**norm(f"{name}.nin_shortcut", p_out),
|
||||
}
|
||||
|
||||
return permutation_spec_from_axes_to_perm(
|
||||
{
|
||||
# Skipped Layers
|
||||
**skip("betas", None, None),
|
||||
**skip("alphas_cumprod", None, None),
|
||||
**skip("alphas_cumprod_prev", None, None),
|
||||
**skip("sqrt_alphas_cumprod", None, None),
|
||||
**skip("sqrt_one_minus_alphas_cumprod", None, None),
|
||||
**skip("log_one_minus_alphas_cumprods", None, None),
|
||||
**skip("sqrt_recip_alphas_cumprod", None, None),
|
||||
**skip("sqrt_recipm1_alphas_cumprod", None, None),
|
||||
**skip("posterior_variance", None, None),
|
||||
**skip("posterior_log_variance_clipped", None, None),
|
||||
**skip("posterior_mean_coef1", None, None),
|
||||
**skip("posterior_mean_coef2", None, None),
|
||||
**skip("log_one_minus_alphas_cumprod", None, None),
|
||||
**skip("model_ema.decay", None, None),
|
||||
**skip("model_ema.num_updates", None, None),
|
||||
# initial
|
||||
**dense("model.diffusion_model.time_embed.0", None, "P_bg0", bias=True),
|
||||
**dense("model.diffusion_model.time_embed.2", "P_bg0", "P_bg1", bias=True),
|
||||
**conv("model.diffusion_model.input_blocks.0.0", "P_bg2", "P_bg3"),
|
||||
# input blocks
|
||||
**easyblock("model.diffusion_model.input_blocks.1.0", "P_bg4", "P_bg5"),
|
||||
**norm("model.diffusion_model.input_blocks.1.1.norm", "P_bg6"),
|
||||
**conv("model.diffusion_model.input_blocks.1.1.proj_in", "P_bg6", "P_bg7"),
|
||||
**dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_q", "P_bg8", "P_bg9", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_k", "P_bg8", "P_bg9", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_v", "P_bg8", "P_bg9", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_out.0", "P_bg8", "P_bg9", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.ff.net.0.proj", "P_bg10", "P_bg11", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.ff.net.2", "P_bg12", "P_bg13", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_q", "P_bg14", "P_bg15", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_k", "P_bg16", "P_bg17", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_v", "P_bg16", "P_bg17", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_out.0", "P_bg18", "P_bg19", bias=True),
|
||||
**norm("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.norm1", "P_bg19"),
|
||||
**norm("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.norm2", "P_bg19"),
|
||||
**norm("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.norm3", "P_bg19"),
|
||||
**conv("model.diffusion_model.input_blocks.1.1.proj_out", "P_bg19", "P_bg20"),
|
||||
**easyblock("model.diffusion_model.input_blocks.2.0", "P_bg21", "P_bg22"),
|
||||
**norm("model.diffusion_model.input_blocks.2.1.norm", "P_bg23"),
|
||||
**conv("model.diffusion_model.input_blocks.2.1.proj_in", "P_bg23", "P_bg24"),
|
||||
**dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn1.to_q", "P_bg25", "P_bg26", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn1.to_k", "P_bg25", "P_bg26", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn1.to_v", "P_bg25", "P_bg26", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn1.to_out.0", "P_bg25", "P_bg26", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.ff.net.0.proj", "P_bg27", "P_bg28", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.ff.net.2", "P_bg29", "P_bg30", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_q", "P_bg31", "P_bg32", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_k", "P_bg33", "P_bg34", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_v", "P_bg33", "P_bg34", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_out.0", "P_bg35", "P_bg36", bias=True),
|
||||
**norm("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.norm1", "P_bg36"),
|
||||
**norm("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.norm2", "P_bg36"),
|
||||
**norm("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.norm3", "P_bg36"),
|
||||
**conv("model.diffusion_model.input_blocks.2.1.proj_out", "P_bg36", "P_bg37"),
|
||||
**conv("model.diffusion_model.input_blocks.3.0.op", "P_bg38", "P_bg39"),
|
||||
**easyblock("model.diffusion_model.input_blocks.4.0", "P_bg40", "P_bg41"),
|
||||
**conv("model.diffusion_model.input_blocks.4.0.skip_connection", "P_bg42", "P_bg43"),
|
||||
**norm("model.diffusion_model.input_blocks.4.1.norm", "P_bg44"),
|
||||
**conv("model.diffusion_model.input_blocks.4.1.proj_in", "P_bg44", "P_bg45"),
|
||||
**dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_q", "P_bg46", "P_bg47", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_k", "P_bg46", "P_bg47", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_v", "P_bg46", "P_bg47", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_out.0", "P_bg46", "P_bg47", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.ff.net.0.proj", "P_bg48", "P_bg49", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.ff.net.2", "P_bg50", "P_bg51", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_q", "P_bg52", "P_bg53", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_k", "P_bg54", "P_bg55", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_v", "P_bg54", "P_bg55", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_out.0", "P_bg56", "P_bg57", bias=True),
|
||||
**norm("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.norm1", "P_bg57"),
|
||||
**norm("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.norm2", "P_bg57"),
|
||||
**norm("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.norm3", "P_bg57"),
|
||||
**conv("model.diffusion_model.input_blocks.4.1.proj_out", "P_bg57", "P_bg58"),
|
||||
**easyblock("model.diffusion_model.input_blocks.5.0", "P_bg59", "P_bg60"),
|
||||
**norm("model.diffusion_model.input_blocks.5.1.norm", "P_bg61"),
|
||||
**conv("model.diffusion_model.input_blocks.5.1.proj_in", "P_bg61", "P_bg62"),
|
||||
**dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_q", "P_bg63", "P_bg64", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_k", "P_bg63", "P_bg64", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_v", "P_bg63", "P_bg64", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_out.0", "P_bg63", "P_bg64", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.ff.net.0.proj", "P_bg65", "P_bg66", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.ff.net.2", "P_bg67", "P_bg68", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_q", "P_bg69", "P_bg70", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_k", "P_bg71", "P_bg72", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_v", "P_bg71", "P_bg72", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_out.0", "P_bg73", "P_bg74", bias=True),
|
||||
**norm("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.norm1", "P_bg74"),
|
||||
**norm("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.norm2", "P_bg74"),
|
||||
**norm("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.norm3", "P_bg74"),
|
||||
**conv("model.diffusion_model.input_blocks.5.1.proj_out", "P_bg74", "P_bg75"),
|
||||
**conv("model.diffusion_model.input_blocks.6.0.op", "P_bg76", "P_bg77"),
|
||||
**easyblock("model.diffusion_model.input_blocks.7.0", "P_bg78", "P_bg79"),
|
||||
**conv("model.diffusion_model.input_blocks.7.0.skip_connection", "P_bg80", "P_bg81"),
|
||||
**norm("model.diffusion_model.input_blocks.7.1.norm", "P_bg82"),
|
||||
**conv("model.diffusion_model.input_blocks.7.1.proj_in", "P_bg82", "P_bg83"),
|
||||
**dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_q", "P_bg84", "P_bg85", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_k", "P_bg84", "P_bg85", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_v", "P_bg84", "P_bg85", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_out.0", "P_bg84", "P_bg85", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.ff.net.0.proj", "P_bg86", "P_bg87", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.ff.net.2", "P_bg88", "P_bg89", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_q", "P_bg90", "P_bg91", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_k", "P_bg92", "P_bg93", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_v", "P_bg92", "P_bg93", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_out.0", "P_bg94", "P_bg95", bias=True),
|
||||
**norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.norm1", "P_bg95"),
|
||||
**norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.norm2", "P_bg95"),
|
||||
**norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.norm3", "P_bg95"),
|
||||
**conv("model.diffusion_model.input_blocks.7.1.proj_out", "P_bg95", "P_bg96"),
|
||||
**easyblock("model.diffusion_model.input_blocks.8.0", "P_bg97", "P_bg98"),
|
||||
**norm("model.diffusion_model.input_blocks.8.1.norm", "P_bg99"),
|
||||
**conv("model.diffusion_model.input_blocks.8.1.proj_in", "P_bg99", "P_bg100"),
|
||||
**dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_q", "P_bg101", "P_bg102", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_k", "P_bg101", "P_bg102", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_v", "P_bg101", "P_bg102", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_out.0", "P_bg101", "P_bg102", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.ff.net.0.proj", "P_bg103", "P_bg104", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.ff.net.2", "P_bg105", "P_bg106", bias=True),
|
||||
**dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_q", "P_bg107", "P_bg108", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_k", "P_bg109", "P_bg110", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_v", "P_bg109", "P_bg110", bias=False),
|
||||
**dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_out.0", "P_bg111", "P_bg112", bias=True),
|
||||
**norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.norm1", "P_bg112"),
|
||||
**norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.norm2", "P_bg112"),
|
||||
**norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.norm3", "P_bg112"),
|
||||
**conv("model.diffusion_model.input_blocks.8.1.proj_out", "P_bg112", "P_bg113"),
|
||||
**conv("model.diffusion_model.input_blocks.9.0.op", "P_bg114", "P_bg115"),
|
||||
**easyblock("model.diffusion_model.input_blocks.10.0", "P_bg115", "P_bg116"),
|
||||
**easyblock("model.diffusion_model.input_blocks.11.0", "P_bg116", "P_bg117"),
|
||||
# middle blocks
|
||||
**easyblock("model.diffusion_model.middle_block.0", "P_bg117", "P_bg118"),
|
||||
**norm("model.diffusion_model.middle_block.1.norm", "P_bg119"),
|
||||
**conv("model.diffusion_model.middle_block.1.proj_in", "P_bg119", "P_bg120"),
|
||||
**dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_q", "P_bg121", "P_bg122", bias=False),
|
||||
**dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_k", "P_bg121", "P_bg122", bias=False),
|
||||
**dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_v", "P_bg121", "P_bg122", bias=False),
|
||||
**dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_out.0", "P_bg121", "P_bg122", bias=True),
|
||||
**dense("model.diffusion_model.middle_block.1.transformer_blocks.0.ff.net.0.proj", "P_bg123", "P_bg124", bias=True),
|
||||
**dense("model.diffusion_model.middle_block.1.transformer_blocks.0.ff.net.2", "P_bg125", "P_bg126", bias=True),
|
||||
**dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_q", "P_bg127", "P_bg128", bias=False),
|
||||
**dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_k", "P_bg129", "P_bg130", bias=False),
|
||||
**dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_v", "P_bg129", "P_bg130", bias=False),
|
||||
**dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_out.0", "P_bg131", "P_bg132", bias=True),
|
||||
**norm("model.diffusion_model.middle_block.1.transformer_blocks.0.norm1", "P_bg132"),
|
||||
**norm("model.diffusion_model.middle_block.1.transformer_blocks.0.norm2", "P_bg132"),
|
||||
**norm("model.diffusion_model.middle_block.1.transformer_blocks.0.norm3", "P_bg132"),
|
||||
**conv("model.diffusion_model.middle_block.1.proj_out", "P_bg132", "P_bg133"),
|
||||
**easyblock("model.diffusion_model.middle_block.2", "P_bg134", "P_bg135"),
|
||||
# output blocks
|
||||
**easyblock("model.diffusion_model.output_blocks.0.0", "P_bg136", "P_bg137"),
|
||||
**conv("model.diffusion_model.output_blocks.0.0.skip_connection", "P_bg138", "P_bg139"),
|
||||
**easyblock("model.diffusion_model.output_blocks.1.0", "P_bg140", "P_bg141"),
|
||||
**conv("model.diffusion_model.output_blocks.1.0.skip_connection", "P_bg142", "P_bg143"),
|
||||
**easyblock("model.diffusion_model.output_blocks.2.0", "P_bg144", "P_bg145"),
|
||||
**conv("model.diffusion_model.output_blocks.2.0.skip_connection", "P_bg146", "P_bg147"),
|
||||
**conv("model.diffusion_model.output_blocks.2.1.conv", "P_bg148", "P_bg149"),
|
||||
**easyblock("model.diffusion_model.output_blocks.3.0", "P_bg150", "P_bg151"),
|
||||
**conv("model.diffusion_model.output_blocks.3.0.skip_connection", "P_bg152", "P_bg153"),
|
||||
**norm("model.diffusion_model.output_blocks.3.1.norm", "P_bg154"),
|
||||
**conv("model.diffusion_model.output_blocks.3.1.proj_in", "P_bg154", "P_bg155"),
|
||||
**dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_q", "P_bg156", "P_bg157", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_k", "P_bg156", "P_bg157", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_v", "P_bg156", "P_bg157", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_out.0", "P_bg156", "P_bg157", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.ff.net.0.proj", "P_bg158", "P_bg159", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.ff.net.2", "P_bg160", "P_bg161", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_q", "P_bg162", "P_bg163", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_k", "P_bg164", "P_bg165", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_v", "P_bg164", "P_bg165", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_out.0", "P_bg166", "P_bg167", bias=True),
|
||||
**norm("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.norm1", "P_bg167"),
|
||||
**norm("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.norm2", "P_bg167"),
|
||||
**norm("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.norm3", "P_bg167"),
|
||||
**conv("model.diffusion_model.output_blocks.3.1.proj_out", "P_bg167", "P_bg168"),
|
||||
**easyblock("model.diffusion_model.output_blocks.4.0", "P_bg169", "P_bg170"),
|
||||
**conv("model.diffusion_model.output_blocks.4.0.skip_connection", "P_bg171", "P_bg172"),
|
||||
**norm("model.diffusion_model.output_blocks.4.1.norm", "P_bg173"),
|
||||
**conv("model.diffusion_model.output_blocks.4.1.proj_in", "P_bg173", "P_bg174"),
|
||||
**dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_q", "P_bg175", "P_bg176", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_k", "P_bg175", "P_bg176", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_v", "P_bg175", "P_bg176", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_out.0", "P_bg175", "P_bg176", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.ff.net.0.proj", "P_bg177", "P_bg178", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.ff.net.2", "P_bg179", "P_bg180", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_q", "P_bg181", "P_bg182", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_k", "P_bg183", "P_bg184", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_v", "P_bg183", "P_bg184", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_out.0", "P_bg185", "P_bg186", bias=True),
|
||||
**norm("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.norm1", "P_bg186"),
|
||||
**norm("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.norm2", "P_bg186"),
|
||||
**norm("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.norm3", "P_bg186"),
|
||||
**conv("model.diffusion_model.output_blocks.4.1.proj_out", "P_bg186", "P_bg187"),
|
||||
**easyblock("model.diffusion_model.output_blocks.5.0", "P_bg188", "P_bg189"),
|
||||
**conv("model.diffusion_model.output_blocks.5.0.skip_connection", "P_bg190", "P_bg191"),
|
||||
**norm("model.diffusion_model.output_blocks.5.1.norm", "P_bg192"),
|
||||
**conv("model.diffusion_model.output_blocks.5.1.proj_in", "P_bg192", "P_bg193"),
|
||||
**dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_q", "P_bg194", "P_bg195", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_k", "P_bg194", "P_bg195", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_v", "P_bg194", "P_bg195", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_out.0", "P_bg194", "P_bg195", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.ff.net.0.proj", "P_bg196", "P_bg197", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.ff.net.2", "P_bg198", "P_bg199", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_q", "P_bg200", "P_bg201", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_k", "P_bg202", "P_bg203", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_v", "P_bg202", "P_bg203", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_out.0", "P_bg204", "P_bg205", bias=True),
|
||||
**norm("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.norm1", "P_bg205"),
|
||||
**norm("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.norm2", "P_bg205"),
|
||||
**norm("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.norm3", "P_bg205"),
|
||||
**conv("model.diffusion_model.output_blocks.5.1.proj_out", "P_bg205", "P_bg206"),
|
||||
**conv("model.diffusion_model.output_blocks.5.2.conv", "P_bg206", "P_bg207"),
|
||||
**easyblock("model.diffusion_model.output_blocks.6.0", "P_bg208", "P_bg209"),
|
||||
**conv("model.diffusion_model.output_blocks.6.0.skip_connection", "P_bg210", "P_bg211"),
|
||||
**norm("model.diffusion_model.output_blocks.6.1.norm", "P_bg212"),
|
||||
**conv("model.diffusion_model.output_blocks.6.1.proj_in", "P_bg212", "P_bg213"),
|
||||
**dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn1.to_q", "P_bg214", "P_bg215", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn1.to_k", "P_bg214", "P_bg215", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn1.to_v", "P_bg214", "P_bg215", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn1.to_out.0", "P_bg214", "P_bg215", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.ff.net.0.proj", "P_bg216", "P_bg217", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.ff.net.2", "P_bg218", "P_bg219", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn2.to_q", "P_bg220", "P_bg221", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn2.to_k", "P_bg222", "P_bg223", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn2.to_v", "P_bg222", "P_bg223", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn2.to_out.0", "P_bg224", "P_bg225", bias=True),
|
||||
**norm("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.norm1", "P_bg225"),
|
||||
**norm("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.norm2", "P_bg225"),
|
||||
**norm("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.norm3", "P_bg225"),
|
||||
**conv("model.diffusion_model.output_blocks.6.1.proj_out", "P_bg225", "P_bg226"),
|
||||
**easyblock("model.diffusion_model.output_blocks.7.0", "P_bg227", "P_bg228"),
|
||||
**conv("model.diffusion_model.output_blocks.7.0.skip_connection", "P_bg229", "P_bg230"),
|
||||
**norm("model.diffusion_model.output_blocks.7.1.norm", "P_bg231"),
|
||||
**conv("model.diffusion_model.output_blocks.7.1.proj_in", "P_bg231", "P_bg232"),
|
||||
**dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_q", "P_bg233", "P_bg234", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_k", "P_bg233", "P_bg234", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_v", "P_bg233", "P_bg234", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_out.0", "P_bg233", "P_bg234", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.ff.net.0.proj", "P_bg235", "P_bg236", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.ff.net.2", "P_bg237", "P_bg238", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn2.to_q", "P_bg239", "P_bg240", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn2.to_k", "P_bg241", "P_bg242", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn2.to_v", "P_bg241", "P_bg242", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn2.to_out.0", "P_bg243", "P_bg244", bias=True),
|
||||
**norm("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.norm1", "P_bg244"),
|
||||
**norm("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.norm2", "P_bg244"),
|
||||
**norm("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.norm3", "P_bg244"),
|
||||
**conv("model.diffusion_model.output_blocks.7.1.proj_out", "P_bg244", "P_bg245"),
|
||||
**easyblock("model.diffusion_model.output_blocks.8.0", "P_bg246", "P_bg247"),
|
||||
**conv("model.diffusion_model.output_blocks.8.0.skip_connection", "P_bg248", "P_bg249"),
|
||||
**norm("model.diffusion_model.output_blocks.8.1.norm", "P_bg250"),
|
||||
**conv("model.diffusion_model.output_blocks.8.1.proj_in", "P_bg250", "P_bg251"),
|
||||
**dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn1.to_q", "P_bg252", "P_bg253", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn1.to_k", "P_bg252", "P_bg253", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn1.to_v", "P_bg252", "P_bg253", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn1.to_out.0", "P_bg252", "P_bg253", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.ff.net.0.proj", "P_bg254", "P_bg255", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.ff.net.2", "P_bg256", "P_bg257", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn2.to_q", "P_bg258", "P_bg259", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn2.to_k", "P_bg260", "P_bg261", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn2.to_v", "P_bg260", "P_bg261", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn2.to_out.0", "P_bg262", "P_bg263", bias=True),
|
||||
**norm("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.norm1", "P_bg263"),
|
||||
**norm("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.norm2", "P_bg263"),
|
||||
**norm("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.norm3", "P_bg263"),
|
||||
**conv("model.diffusion_model.output_blocks.8.1.proj_out", "P_bg263", "P_bg264"),
|
||||
**conv("model.diffusion_model.output_blocks.8.2.conv", "P_bg265", "P_bg266"),
|
||||
**easyblock("model.diffusion_model.output_blocks.9.0", "P_bg267", "P_bg268"),
|
||||
**conv("model.diffusion_model.output_blocks.9.0.skip_connection", "P_bg269", "P_bg270"),
|
||||
**norm("model.diffusion_model.output_blocks.9.1.norm", "P_bg271"),
|
||||
**conv("model.diffusion_model.output_blocks.9.1.proj_in", "P_bg271", "P_bg272"),
|
||||
**dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn1.to_q", "P_bg273", "P_bg274", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn1.to_k", "P_bg273", "P_bg274", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn1.to_v", "P_bg273", "P_bg274", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn1.to_out.0", "P_bg273", "P_bg274", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.ff.net.0.proj", "P_bg275", "P_bg276", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.ff.net.2", "P_bg277", "P_bg278", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn2.to_q", "P_bg279", "P_bg280", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn2.to_k", "P_bg281", "P_bg282", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn2.to_v", "P_bg281", "P_bg282", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn2.to_out.0", "P_bg283", "P_bg284", bias=True),
|
||||
**norm("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.norm1", "P_bg284"),
|
||||
**norm("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.norm2", "P_bg284"),
|
||||
**norm("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.norm3", "P_bg284"),
|
||||
**conv("model.diffusion_model.output_blocks.9.1.proj_out", "P_bg284", "P_bg285"),
|
||||
**easyblock("model.diffusion_model.output_blocks.10.0", "P_bg286", "P_bg287"),
|
||||
**conv("model.diffusion_model.output_blocks.10.0.skip_connection", "P_bg288", "P_bg289"),
|
||||
**norm("model.diffusion_model.output_blocks.10.1.norm", "P_bg290"),
|
||||
**conv("model.diffusion_model.output_blocks.10.1.proj_in", "P_bg290", "P_bg291"),
|
||||
**dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn1.to_q", "P_bg292", "P_bg293", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn1.to_k", "P_bg292", "P_bg293", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn1.to_v", "P_bg292", "P_bg293", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn1.to_out.0", "P_bg292", "P_bg293", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.ff.net.0.proj", "P_b294", "P_bg295", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.ff.net.2", "P_bg296", "P_bg297", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn2.to_q", "P_bg298", "P_bg299", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn2.to_k", "P_bg300", "P_bg301", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn2.to_v", "P_bg300", "P_bg301", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn2.to_out.0", "P_bg302", "P_bg303", bias=True),
|
||||
**norm("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.norm1", "P_bg303"),
|
||||
**norm("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.norm2", "P_bg303"),
|
||||
**norm("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.norm3", "P_bg303"),
|
||||
**conv("model.diffusion_model.output_blocks.10.1.proj_out", "P_bg303", "P_bg304"),
|
||||
**easyblock("model.diffusion_model.output_blocks.11.0", "P_bg305", "P_bg306"),
|
||||
**conv("model.diffusion_model.output_blocks.11.0.skip_connection", "P_bg307", "P_bg308"),
|
||||
**norm("model.diffusion_model.output_blocks.11.1.norm", "P_bg309"),
|
||||
**conv("model.diffusion_model.output_blocks.11.1.proj_in", "P_bg309", "P_bg310"),
|
||||
**dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn1.to_q", "P_bg311", "P_bg312", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn1.to_k", "P_bg311", "P_bg312", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn1.to_v", "P_bg311", "P_bg312", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn1.to_out.0", "P_bg311", "P_bg312", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.ff.net.0.proj", "P_bg313", "P_bg314", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.ff.net.2", "P_bg315", "P_bg316", bias=True),
|
||||
**dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn2.to_q", "P_bg317", "P_bg318", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn2.to_k", "P_bg319", "P_bg320", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn2.to_v", "P_bg319", "P_bg320", bias=False),
|
||||
**dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn2.to_out.0", "P_bg321", "P_bg322", bias=True),
|
||||
**norm("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.norm1", "P_bg322"),
|
||||
**norm("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.norm2", "P_bg322"),
|
||||
**norm("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.norm3", "P_bg322"),
|
||||
**conv("model.diffusion_model.output_blocks.11.1.proj_out", "P_bg322", "P_bg323"),
|
||||
**norm("model.diffusion_model.out.0", "P_bg324"),
|
||||
**conv("model.diffusion_model.out.2", "P_bg325", "P_bg326"),
|
||||
**skip("cond_stage_model.transformer.text_model.embeddings.position_ids", None, None),
|
||||
**dense("cond_stage_model.transformer.text_model.embeddings.token_embedding", "P_bg365", "P_bg366", bias=False),
|
||||
**dense("cond_stage_model.transformer.text_model.embeddings.token_embedding", None, None),
|
||||
**dense("cond_stage_model.transformer.text_model.embeddings.position_embedding", "P_bg367", "P_bg368", bias=False),
|
||||
# cond stage text encoder
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.0.self_attn.k_proj", "P_bg369", "P_bg370", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.0.self_attn.v_proj", "P_bg369", "P_bg370", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.0.self_attn.q_proj", "P_bg369", "P_bg370", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.0.self_attn.out_proj", "P_bg369", "P_bg370", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.0.layer_norm1", "P_bg370"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.0.mlp.fc1", "P_bg370", "P_bg371", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.0.mlp.fc2", "P_bg371", "P_bg372", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.0.layer_norm2", "P_bg372"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.1.self_attn.k_proj", "P_bg372", "P_bg373", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.1.self_attn.v_proj", "P_bg372", "P_bg373", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.1.self_attn.q_proj", "P_bg372", "P_bg373", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.1.self_attn.out_proj", "P_bg372", "P_bg373", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.1.layer_norm1", "P_bg373"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.1.mlp.fc1", "P_bg373", "P_bg374", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.1.mlp.fc2", "P_bg374", "P_bg375", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.1.layer_norm2", "P_bg375"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.2.self_attn.k_proj", "P_bg375", "P_bg376", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.2.self_attn.v_proj", "P_bg375", "P_bg376", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.2.self_attn.q_proj", "P_bg375", "P_bg376", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.2.self_attn.out_proj", "P_bg375", "P_bg376", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.2.layer_norm1", "P_bg376"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.2.mlp.fc1", "P_bg376", "P_bg377", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.2.mlp.fc2", "P_bg377", "P_bg378", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.2.layer_norm2", "P_bg378"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.3.self_attn.k_proj", "P_bg378", "P_bg379", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.3.self_attn.v_proj", "P_bg378", "P_bg379", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.3.self_attn.q_proj", "P_bg378", "P_bg379", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.3.self_attn.out_proj", "P_bg378", "P_bg379", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.3.layer_norm1", "P_bg379"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.3.mlp.fc1", "P_bg379", "P_bg380", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.3.mlp.fc2", "P_bg380", "P_b381", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.3.layer_norm2", "P_bg381"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.4.self_attn.k_proj", "P_bg381", "P_bg382", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.4.self_attn.v_proj", "P_bg381", "P_bg382", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.4.self_attn.q_proj", "P_bg381", "P_bg382", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.4.self_attn.out_proj", "P_bg381", "P_bg382", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.4.layer_norm1", "P_bg382"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.4.mlp.fc1", "P_bg382", "P_bg383", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.4.mlp.fc2", "P_bg383", "P_bg384", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.4.layer_norm2", "P_bg384"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.5.self_attn.k_proj", "P_bg384", "P_bg385", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.5.self_attn.v_proj", "P_bg384", "P_bg385", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.5.self_attn.q_proj", "P_bg384", "P_bg385", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.5.self_attn.out_proj", "P_bg384", "P_bg385", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.5.layer_norm1", "P_bg385"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.5.mlp.fc1", "P_bg385", "P_bg386", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.5.mlp.fc2", "P_bg386", "P_bg387", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.5.layer_norm2", "P_bg387"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.6.self_attn.k_proj", "P_bg387", "P_bg388", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.6.self_attn.v_proj", "P_bg387", "P_bg388", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.6.self_attn.q_proj", "P_bg387", "P_bg388", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.6.self_attn.out_proj", "P_bg387", "P_bg388", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.6.layer_norm1", "P_bg389"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.6.mlp.fc1", "P_bg389", "P_bg390", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.6.mlp.fc2", "P_bg390", "P_bg391", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.6.layer_norm2", "P_bg391"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.7.self_attn.k_proj", "P_bg391", "P_bg392", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.7.self_attn.v_proj", "P_bg391", "P_bg392", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.7.self_attn.q_proj", "P_bg391", "P_bg392", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.7.self_attn.out_proj", "P_bg391", "P_bg392", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.7.layer_norm1", "P_bg392"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.7.mlp.fc1", "P_bg392", "P_bg393", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.7.mlp.fc2", "P_bg393", "P_bg394", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.7.layer_norm2", "P_bg394"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.8.self_attn.k_proj", "P_bg394", "P_bg395", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.8.self_attn.v_proj", "P_bg394", "P_bg395", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.8.self_attn.q_proj", "P_bg394", "P_bg395", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.8.self_attn.out_proj", "P_bg394", "P_bg395", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.8.layer_norm1", "P_bg395"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.8.mlp.fc1", "P_bg395", "P_bg396", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.8.mlp.fc2", "P_bg396", "P_bg397", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.8.layer_norm2", "P_bg397"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.9.self_attn.k_proj", "P_bg397", "P_bg398", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.9.self_attn.v_proj", "P_bg397", "P_bg398", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.9.self_attn.q_proj", "P_bg397", "P_bg398", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.9.self_attn.out_proj", "P_bg397", "P_bg398", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.9.layer_norm1", "P_bg398"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.9.mlp.fc1", "P_bg398", "P_bg399", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.9.mlp.fc2", "P_bg400", "P_bg401", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.9.layer_norm2", "P_bg401"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.10.self_attn.k_proj", "P_bg401", "P_bg402", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.10.self_attn.v_proj", "P_bg401", "P_bg402", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.10.self_attn.q_proj", "P_bg401", "P_bg402", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.10.self_attn.out_proj", "P_bg401", "P_bg402", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.10.layer_norm1", "P_bg402"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.10.mlp.fc1", "P_bg402", "P_bg403", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.10.mlp.fc2", "P_bg403", "P_bg404", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.10.layer_norm2", "P_bg404"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.11.self_attn.k_proj", "P_bg404", "P_bg405", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.11.self_attn.v_proj", "P_bg404", "P_bg405", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.11.self_attn.q_proj", "P_bg404", "P_bg405", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.11.self_attn.out_proj", "P_bg404", "P_bg405", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.11.layer_norm1", "P_bg405"),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.11.mlp.fc1", "P_bg405", "P_bg406", bias=True),
|
||||
**dense("cond_stage_model.transformer.text_model.encoder.layers.11.mlp.fc2", "P_bg406", "P_bg407", bias=True),
|
||||
**norm("cond_stage_model.transformer.text_model.encoder.layers.11.layer_norm2", "P_bg407"),
|
||||
**norm("cond_stage_model.transformer.text_model.final_layer_norm", "P_bg407"),
|
||||
}
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -205,7 +205,7 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config
|
||||
download_config["mirror"] = mirror
|
||||
if custom_pipeline is not None and len(custom_pipeline) > 0:
|
||||
download_config["custom_pipeline"] = custom_pipeline
|
||||
shared.log.debug(f"Diffusers downloading: {hub_id} {download_config}")
|
||||
shared.log.debug(f"Diffusers downloading: {hub_id} args={download_config}")
|
||||
if token is not None and len(token) > 2:
|
||||
shared.log.debug(f"Diffusers authentication: {token}")
|
||||
hf.login(token)
|
||||
@@ -574,6 +574,7 @@ def move_files(src_path: str, dest_path: str, ext_filter: str = None):
|
||||
|
||||
def load_upscalers():
|
||||
# We can only do this 'magic' method to dynamically load upscalers if they are referenced, so we'll try to import any _model.py files before looking in __subclasses__
|
||||
t0 = time.time()
|
||||
modules_dir = os.path.join(shared.script_path, "modules", "postprocess")
|
||||
for file in os.listdir(modules_dir):
|
||||
if "_model.py" in file:
|
||||
@@ -602,4 +603,5 @@ def load_upscalers():
|
||||
datas += scaler.scalers
|
||||
names.append(name[8:])
|
||||
shared.sd_upscalers = sorted(datas, key=lambda x: x.name.lower() if not isinstance(x.scaler, (UpscalerNone, UpscalerLanczos, UpscalerNearest)) else "") # Special case for UpscalerNone keeps it at the beginning of the list.
|
||||
shared.log.debug(f"Load upscalers: total={len(shared.sd_upscalers)} downloaded={len([x for x in shared.sd_upscalers if x.data_path is not None and os.path.isfile(x.data_path)])} user={len([x for x in shared.sd_upscalers if x.custom])} {names}")
|
||||
t1 = time.time()
|
||||
shared.log.debug(f"Load upscalers: total={len(shared.sd_upscalers)} downloaded={len([x for x in shared.sd_upscalers if x.data_path is not None and os.path.isfile(x.data_path)])} user={len([x for x in shared.sd_upscalers if x.custom])} time={t1-t0:.2f} {names}")
|
||||
|
||||
+51
-49
@@ -18,7 +18,7 @@ from ldm.models.diffusion.ddpm import LatentDepth2ImageDiffusion
|
||||
from einops import repeat, rearrange
|
||||
from blendmodes.blend import blendLayers, BlendType
|
||||
from installer import git_commit
|
||||
from modules import shared, devices
|
||||
from modules import shared, devices, errors
|
||||
import modules.memstats
|
||||
import modules.lowvram
|
||||
import modules.masking
|
||||
@@ -121,8 +121,7 @@ class StableDiffusionProcessing:
|
||||
"""
|
||||
The first set of paramaters: sd_models -> do_not_reload_embeddings represent the minimum required to create a StableDiffusionProcessing
|
||||
"""
|
||||
def __init__(self, sd_model=None, outpath_samples=None, outpath_grids=None, prompt: str = "", styles: List[str] = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, seed_enable_extras: bool = True, sampler_name: str = None, latent_sampler: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, cfg_scale: float = 7.0, image_cfg_scale: float = None, clip_skip: int = 1, width: int = 512, height: int = 512, full_quality: bool = True, restore_faces: bool = False, tiling: bool = False, do_not_save_samples: bool = False, do_not_save_grid: bool = False, extra_generation_params: Dict[Any, Any] = None, overlay_images: Any = None, negative_prompt: str = None, eta: float = None, do_not_reload_embeddings: bool = False, denoising_strength: float = 0, diffusers_guidance_rescale: float = 0.7, override_settings: Dict[str, Any] = None, override_settings_restore_afterwards: bool = True, sampler_index: int = None, script_args: list = None): # pylint: disable=unused-argument
|
||||
|
||||
def __init__(self, sd_model=None, outpath_samples=None, outpath_grids=None, prompt: str = "", styles: List[str] = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, seed_enable_extras: bool = True, sampler_name: str = None, latent_sampler: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, cfg_scale: float = 7.0, image_cfg_scale: float = None, clip_skip: int = 1, width: int = 512, height: int = 512, full_quality: bool = True, restore_faces: bool = False, tiling: bool = False, do_not_save_samples: bool = False, do_not_save_grid: bool = False, extra_generation_params: Dict[Any, Any] = None, overlay_images: Any = None, negative_prompt: str = None, eta: float = None, do_not_reload_embeddings: bool = False, denoising_strength: float = 0, diffusers_guidance_rescale: float = 0.7, hdr_clamp: bool = False, hdr_boundary: float = 4.0, hdr_threshold: float = 3.5, hdr_center: bool = False, hdr_channel_shift: float = 0.8, hdr_full_shift: float = 0.8, hdr_maximize: bool = False, hdr_max_center: float = 0.6, hdr_max_boundry: float = 1.0, override_settings: Dict[str, Any] = None, override_settings_restore_afterwards: bool = True, sampler_index: int = None, script_args: list = None): # pylint: disable=unused-argument
|
||||
self.outpath_samples: str = outpath_samples
|
||||
self.outpath_grids: str = outpath_grids
|
||||
self.prompt: str = prompt
|
||||
@@ -166,8 +165,8 @@ class StableDiffusionProcessing:
|
||||
self.disable_extra_networks = False
|
||||
self.token_merging_ratio = 0
|
||||
self.token_merging_ratio_hr = 0
|
||||
self.scripts = None
|
||||
self.script_args = script_args or []
|
||||
# self.scripts = modules.scripts.ScriptRunner() # set via property
|
||||
# self.script_args = script_args or [] # set via property
|
||||
self.per_script_args = {}
|
||||
self.all_prompts = None
|
||||
self.all_negative_prompts = None
|
||||
@@ -192,6 +191,7 @@ class StableDiffusionProcessing:
|
||||
self.s_tmin = shared.opts.s_tmin
|
||||
self.s_tmax = float('inf') # not representable as a standard ui option
|
||||
shared.opts.data['clip_skip'] = clip_skip
|
||||
self.task_args = {}
|
||||
# TODO a1111 compatibility items
|
||||
self.refiner_switch_at = 0
|
||||
self.hr_prompt = ''
|
||||
@@ -203,6 +203,16 @@ class StableDiffusionProcessing:
|
||||
self.scripts_value: modules.scripts.ScriptRunner = field(default=None, init=False)
|
||||
self.script_args_value: list = field(default=None, init=False)
|
||||
self.scripts_setup_complete: bool = field(default=False, init=False)
|
||||
# hdr
|
||||
self.hdr_clamp = hdr_clamp
|
||||
self.hdr_boundary = hdr_boundary
|
||||
self.hdr_threshold = hdr_threshold
|
||||
self.hdr_center = hdr_center
|
||||
self.hdr_channel_shift = hdr_channel_shift
|
||||
self.hdr_full_shift = hdr_full_shift
|
||||
self.hdr_maximize = hdr_maximize
|
||||
self.hdr_max_center = hdr_max_center
|
||||
self.hdr_max_boundry = hdr_max_boundry
|
||||
|
||||
|
||||
@property
|
||||
@@ -533,12 +543,20 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
|
||||
index = position_in_batch + iteration * p.batch_size
|
||||
if all_prompts is None:
|
||||
all_prompts = p.all_prompts
|
||||
if all_negative_prompts is None:
|
||||
all_negative_prompts = p.all_negative_prompts
|
||||
if all_seeds is None:
|
||||
all_seeds = p.all_seeds
|
||||
if all_subseeds is None:
|
||||
all_subseeds = p.all_subseeds
|
||||
if all_negative_prompts is None:
|
||||
all_negative_prompts = p.all_negative_prompts
|
||||
while len(all_prompts) <= index:
|
||||
all_prompts.append(all_prompts[-1])
|
||||
while len(all_seeds) <= index:
|
||||
all_seeds.append(all_seeds[-1])
|
||||
while len(all_subseeds) <= index:
|
||||
all_subseeds.append(all_subseeds[-1])
|
||||
while len(all_negative_prompts) <= index:
|
||||
all_negative_prompts.append(all_negative_prompts[-1])
|
||||
comment = ', '.join(comments) if comments is not None and type(comments) is list else None
|
||||
ops = list(set(p.ops))
|
||||
ops.reverse()
|
||||
@@ -597,10 +615,14 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
|
||||
args["Init image size"] = f"{getattr(p, 'init_img_width', 0)}x{getattr(p, 'init_img_height', 0)}"
|
||||
args["Init image hash"] = getattr(p, 'init_img_hash', None)
|
||||
args["Mask weight"] = getattr(p, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) if p.is_using_inpainting_conditioning else None
|
||||
args['Resize mode'] = getattr(p, 'resize_mode', None)
|
||||
args['Resize scale'] = getattr(p, 'scale_by', None)
|
||||
args["Mask blur"] = p.mask_blur if getattr(p, 'mask', None) is not None and getattr(p, 'mask_blur', 0) > 0 else None
|
||||
args["Denoising strength"] = getattr(p, 'denoising_strength', None)
|
||||
# lookup by index
|
||||
if getattr(p, 'resize_mode', None) is not None:
|
||||
RESIZE_MODES = ["None", "Resize fixed", "Crop and resize", "Resize and fill", "Latent upscale"]
|
||||
args['Resize mode'] = RESIZE_MODES[p.resize_mode]
|
||||
# TODO missing-by-index: inpainting_fill, inpaint_full_res, inpainting_mask_invert
|
||||
if 'face' in p.ops:
|
||||
args["Face restoration"] = shared.opts.face_restoration_model
|
||||
if 'color' in p.ops:
|
||||
@@ -644,36 +666,6 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
|
||||
return infotext
|
||||
|
||||
|
||||
"""
|
||||
def print_profile(profile, msg: str):
|
||||
try:
|
||||
from rich import print # pylint: disable=redefined-builtin
|
||||
except Exception:
|
||||
pass
|
||||
lines = profile.key_averages().table(sort_by="cuda_time_total", row_limit=20)
|
||||
lines = lines.split('\n')
|
||||
lines = [l for l in lines if '/profiler' not in l]
|
||||
print(f'Profile {msg}:', '\n'.join(lines))
|
||||
"""
|
||||
|
||||
|
||||
def print_profile(profile, msg: str):
|
||||
import io
|
||||
import pstats
|
||||
try:
|
||||
from rich import print # pylint: disable=redefined-builtin
|
||||
except Exception:
|
||||
pass
|
||||
profile.disable()
|
||||
stream = io.StringIO() # pylint: disable=abstract-class-instantiated
|
||||
ps = pstats.Stats(profile, stream=stream)
|
||||
ps.sort_stats(pstats.SortKey.CUMULATIVE).print_stats(15)
|
||||
profile = None
|
||||
lines = stream.getvalue().split('\n')
|
||||
lines = [line for line in lines if '<frozen' not in line and '{built-in' not in line and '/logging' not in line and '/rich' not in line]
|
||||
print(f'Profile {msg}:', '\n'.join(lines))
|
||||
|
||||
|
||||
def process_images(p: StableDiffusionProcessing) -> Processed:
|
||||
if not hasattr(p.sd_model, 'sd_checkpoint_info'):
|
||||
return None
|
||||
@@ -721,22 +713,26 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
|
||||
modules.script_callbacks.before_process_callback(p)
|
||||
|
||||
if shared.cmd_opts.profile:
|
||||
"""
|
||||
import torch.profiler # pylint: disable=redefined-outer-name
|
||||
with torch.profiler.profile(profile_memory=True, with_modules=True) as prof:
|
||||
with torch.profiler.record_function("process_images"):
|
||||
res = process_images_inner(p)
|
||||
print_profile(prof, 'process_images')
|
||||
"""
|
||||
import cProfile
|
||||
pr = cProfile.Profile()
|
||||
pr.enable()
|
||||
profile_python = cProfile.Profile()
|
||||
profile_python.enable()
|
||||
with context_hypertile_vae(p), context_hypertile_unet(p):
|
||||
import torch.profiler # pylint: disable=redefined-outer-name
|
||||
activities=[torch.profiler.ProfilerActivity.CPU]
|
||||
if torch.cuda.is_available():
|
||||
activities.append(torch.profiler.ProfilerActivity.CUDA)
|
||||
shared.log.debug(f'Torch profile: activities={activities}')
|
||||
if shared.profiler is None:
|
||||
shared.profiler = torch.profiler.profile(activities=activities, profile_memory=True, with_modules=True)
|
||||
shared.profiler.start()
|
||||
shared.profiler.step()
|
||||
res = process_images_inner(p)
|
||||
print_profile(pr, 'Torch')
|
||||
errors.profile_torch(shared.profiler, 'Process')
|
||||
errors.profile(profile_python, 'Process')
|
||||
else:
|
||||
with context_hypertile_vae(p), context_hypertile_unet(p):
|
||||
res = process_images_inner(p)
|
||||
|
||||
finally:
|
||||
if not shared.opts.cuda_compile:
|
||||
modules.sd_models.apply_token_merging(p.sd_model, 0)
|
||||
@@ -754,12 +750,14 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
|
||||
|
||||
|
||||
def validate_sample(tensor):
|
||||
if not isinstance(tensor, np.ndarray) and not isinstance(tensor, torch.Tensor):
|
||||
return tensor
|
||||
if tensor.dtype == torch.bfloat16: # numpy does not support bf16
|
||||
tensor = tensor.to(torch.float16)
|
||||
if shared.backend == shared.Backend.ORIGINAL:
|
||||
sample = 255.0 * np.moveaxis(tensor.cpu().numpy(), 0, 2)
|
||||
else:
|
||||
sample = 255. * tensor
|
||||
sample = 255.0 * tensor
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
cast = sample.astype(np.uint8)
|
||||
if len(w) > 0:
|
||||
@@ -1045,6 +1043,8 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
|
||||
self.refiner_prompt = refiner_prompt
|
||||
self.refiner_negative = refiner_negative
|
||||
self.sampler = None
|
||||
self.scripts = None
|
||||
self.script_args = []
|
||||
|
||||
def init(self, all_prompts, all_seeds, all_subseeds):
|
||||
if shared.backend == shared.Backend.DIFFUSERS:
|
||||
@@ -1207,6 +1207,8 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
|
||||
self.is_batch = False
|
||||
self.scale_by = 1.0
|
||||
self.sampler = None
|
||||
self.scripts = None
|
||||
self.script_args = []
|
||||
|
||||
def init(self, all_prompts, all_seeds, all_subseeds):
|
||||
if shared.backend == shared.Backend.DIFFUSERS and self.image_mask is not None:
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""
|
||||
based on article by TimothyAlexisVass
|
||||
https://huggingface.co/blog/TimothyAlexisVass/explaining-the-sdxl-latent-space
|
||||
"""
|
||||
|
||||
import os
|
||||
import torch
|
||||
from modules import shared
|
||||
|
||||
|
||||
debug = shared.log.info if os.environ.get('SD_HDR_DEBUG', None) is not None else lambda *args, **kwargs: None
|
||||
|
||||
|
||||
def soft_clamp_tensor(input_tensor, threshold=0.8, boundary=4):
|
||||
# shrinking towards the mean; will also remove outliers
|
||||
if max(abs(input_tensor.max()), abs(input_tensor.min())) < boundary or threshold == 0:
|
||||
return input_tensor
|
||||
channel_dim = 1
|
||||
threshold *= boundary
|
||||
max_vals = input_tensor.max(channel_dim, keepdim=True)[0]
|
||||
max_replace = ((input_tensor - threshold) / (max_vals - threshold)) * (boundary - threshold) + threshold
|
||||
over_mask = input_tensor > threshold
|
||||
min_vals = input_tensor.min(channel_dim, keepdim=True)[0]
|
||||
min_replace = ((input_tensor + threshold) / (min_vals + threshold)) * (-boundary + threshold) - threshold
|
||||
under_mask = input_tensor < -threshold
|
||||
debug(f'HDE soft clamp: threshold={threshold} boundary={boundary}')
|
||||
input_tensor = torch.where(over_mask, max_replace, torch.where(under_mask, min_replace, input_tensor))
|
||||
return input_tensor
|
||||
|
||||
|
||||
def center_tensor(input_tensor, channel_shift=1.0, full_shift=1.0, channels=[0, 1, 2, 3]): # pylint: disable=dangerous-default-value # noqa: B006
|
||||
if channel_shift == 0 and full_shift == 0:
|
||||
return input_tensor
|
||||
means = []
|
||||
for channel in channels:
|
||||
means.append(input_tensor[0, channel].mean())
|
||||
input_tensor[0, channel] -= means[-1] * channel_shift
|
||||
debug(f'HDR center: channel-shift{channel_shift} full-shift={full_shift} means={torch.stack(means)}')
|
||||
input_tensor = input_tensor - input_tensor.mean() * full_shift
|
||||
return input_tensor
|
||||
|
||||
|
||||
def maximize_tensor(input_tensor, boundary=1.0, channels=[0, 1, 2]): # pylint: disable=dangerous-default-value # noqa: B006
|
||||
if boundary == 1.0:
|
||||
return input_tensor
|
||||
boundary *= 4
|
||||
min_val = input_tensor.min()
|
||||
max_val = input_tensor.max()
|
||||
normalization_factor = boundary / max(abs(min_val), abs(max_val))
|
||||
input_tensor[0, channels] *= normalization_factor
|
||||
debug(f'HDR maximize: boundary={boundary} min={min_val} max={max_val} factor={normalization_factor}')
|
||||
return input_tensor
|
||||
|
||||
|
||||
def correction_callback(p, timestep, kwags):
|
||||
if timestep > 950 and p.hdr_clamp:
|
||||
kwags["latents"] = soft_clamp_tensor(kwags["latents"], threshold=p.hdr_threshold, boundary=p.hdr_boundary)
|
||||
if timestep > 700 and p.hdr_center:
|
||||
kwags["latents"] = center_tensor(kwags["latents"], channel_shift=p.hdr_channel_shift, full_shift=p.hdr_full_shift)
|
||||
if timestep > 1 and timestep < 100 and p.hdr_maximize:
|
||||
kwags["latents"] = center_tensor(kwags["latents"], channel_shift=p.hdr_max_center, full_shift=1.0)
|
||||
kwags["latents"] = maximize_tensor(kwags["latents"], boundary=p.hdr_max_boundry)
|
||||
return kwags
|
||||
@@ -5,7 +5,6 @@ import inspect
|
||||
import typing
|
||||
import torch
|
||||
import torchvision.transforms.functional as TF
|
||||
import diffusers
|
||||
import modules.devices as devices
|
||||
import modules.shared as shared
|
||||
import modules.sd_samplers as sd_samplers
|
||||
@@ -17,6 +16,7 @@ import modules.errors as errors
|
||||
from modules.processing import StableDiffusionProcessing, create_random_tensors
|
||||
import modules.prompt_parser_diffusers as prompt_parser_diffusers
|
||||
from modules.sd_hijack_hypertile import hypertile_set
|
||||
from modules.processing_correction import correction_callback
|
||||
|
||||
|
||||
def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_prompts):
|
||||
@@ -71,10 +71,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
raise AssertionError('Interrupted...')
|
||||
time.sleep(0.1)
|
||||
|
||||
def diffusers_callback(_pipe, step: int, _timestep: int, kwargs: dict):
|
||||
latents = kwargs['latents']
|
||||
def diffusers_callback(_pipe, step: int, timestep: int, kwargs: dict):
|
||||
shared.state.sampling_step = step
|
||||
shared.state.current_latent = latents
|
||||
if shared.state.interrupted or shared.state.skipped:
|
||||
raise AssertionError('Interrupted...')
|
||||
if shared.state.paused:
|
||||
@@ -83,7 +81,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
if shared.state.interrupted or shared.state.skipped:
|
||||
raise AssertionError('Interrupted...')
|
||||
time.sleep(0.1)
|
||||
return {'latents': latents}
|
||||
if kwargs.get('latents', None) is None:
|
||||
return kwargs
|
||||
kwargs = correction_callback(p, timestep, kwargs)
|
||||
shared.state.current_latent = kwargs['latents']
|
||||
if shared.cmd_opts.profile and shared.profiler is not None:
|
||||
shared.profiler.step()
|
||||
return kwargs
|
||||
|
||||
def full_vae_decode(latents, model):
|
||||
t0 = time.time()
|
||||
@@ -127,7 +131,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
if len(latents) == 0:
|
||||
return []
|
||||
decoded = torch.zeros((len(latents), 3, latents.shape[2] * 8, latents.shape[3] * 8), dtype=devices.dtype_vae, device=devices.device)
|
||||
for i in range(len(output.images)):
|
||||
for i in range(latents.shape[0]):
|
||||
decoded[i] = sd_vae_taesd.decode(latents[i])
|
||||
return decoded
|
||||
|
||||
@@ -137,6 +141,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
return encoded
|
||||
|
||||
def vae_decode(latents, model, output_type='np', full_quality=True):
|
||||
t0 = time.time()
|
||||
prev_job = shared.state.job
|
||||
shared.state.job = 'vae'
|
||||
if not torch.is_tensor(latents): # already decoded
|
||||
@@ -149,6 +154,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
if not hasattr(model, 'vae'):
|
||||
shared.log.error('VAE not found in model')
|
||||
return []
|
||||
if latents.shape[0] == 4 and latents.shape[1] != 4: # likely animatediff latent
|
||||
latents = latents.permute(1, 0, 2, 3)
|
||||
if len(latents.shape) == 3: # lost a batch dim in hires
|
||||
latents = latents.unsqueeze(0)
|
||||
if full_quality:
|
||||
@@ -159,6 +166,9 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
# decoded = validate_sample(decoded)
|
||||
imgs = model.image_processor.postprocess(decoded, output_type=output_type)
|
||||
shared.state.job = prev_job
|
||||
if shared.cmd_opts.profile:
|
||||
t1 = time.time()
|
||||
shared.log.debug(f'Profile: VAE decode: {t1-t0:.2f}')
|
||||
return imgs
|
||||
|
||||
def vae_encode(image, model, full_quality=True): # pylint: disable=unused-variable
|
||||
@@ -198,16 +208,27 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
|
||||
def task_specific_kwargs(model):
|
||||
task_args = {}
|
||||
is_img2img_model = bool("Zero123" in shared.sd_model.__class__.__name__)
|
||||
is_img2img_model = bool('Zero123' in shared.sd_model.__class__.__name__)
|
||||
if sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE and not is_img2img_model:
|
||||
p.ops.append('txt2img')
|
||||
task_args = {"height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8)}
|
||||
task_args = {
|
||||
'height': 8 * math.ceil(p.height / 8),
|
||||
'width': 8 * math.ceil(p.width / 8),
|
||||
}
|
||||
elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or is_img2img_model) and len(getattr(p, 'init_images' ,[])) > 0:
|
||||
p.ops.append('img2img')
|
||||
task_args = {"image": p.init_images, "strength": p.denoising_strength}
|
||||
task_args = {
|
||||
'image': p.init_images,
|
||||
'strength': p.denoising_strength,
|
||||
}
|
||||
elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INSTRUCT and len(getattr(p, 'init_images' ,[])) > 0:
|
||||
p.ops.append('instruct')
|
||||
task_args = {"height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8), "image": p.init_images, "strength": p.denoising_strength}
|
||||
task_args = {
|
||||
'height': 8 * math.ceil(p.height / 8),
|
||||
'width': 8 * math.ceil(p.width / 8),
|
||||
'image': p.init_images,
|
||||
'strength': p.denoising_strength,
|
||||
}
|
||||
elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INPAINTING or is_img2img_model) and len(getattr(p, 'init_images' ,[])) > 0:
|
||||
p.ops.append('inpaint')
|
||||
if getattr(p, 'mask', None) is None:
|
||||
@@ -215,7 +236,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
width = 8 * math.ceil(p.init_images[0].width / 8)
|
||||
height = 8 * math.ceil(p.init_images[0].height / 8)
|
||||
# option-1: use images as inputs
|
||||
task_args = {"image": p.init_images, "mask_image": p.mask, "strength": p.denoising_strength, "height": height, "width": width}
|
||||
task_args = {
|
||||
'image': p.init_images,
|
||||
'mask_image': p.mask,
|
||||
'strength': p.denoising_strength,
|
||||
'height': height,
|
||||
'width': width,
|
||||
}
|
||||
""" # option-2: preprocess images into latents using diffusers
|
||||
vae_scale_factor = 2 ** (len(model.vae.config.block_out_channels) - 1)
|
||||
image_processor = diffusers.image_processor.VaeImageProcessor(vae_scale_factor=vae_scale_factor)
|
||||
@@ -235,14 +262,20 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
task_args = {"image": p.init_images, "mask_image": mask_image, "masked_image_latents": masked_image_latents, "strength": p.denoising_strength, "height": height, "width": width}
|
||||
"""
|
||||
if model.__class__.__name__ == 'LatentConsistencyModelPipeline' and hasattr(p, 'init_images') and len(p.init_images) > 0:
|
||||
p.ops.append('lcm')
|
||||
init_latents = [vae_encode(image, model=shared.sd_model, full_quality=p.full_quality).squeeze(dim=0) for image in p.init_images]
|
||||
init_latent = torch.stack(init_latents, dim=0).to(shared.device)
|
||||
init_noise = p.denoising_strength * create_random_tensors(init_latent.shape[1:], seeds=p.all_seeds, subseeds=p.all_subseeds, subseed_strength=p.subseed_strength, p=p)
|
||||
init_latent = (1 - p.denoising_strength) * init_latent + init_noise
|
||||
task_args = {"latents": init_latent.to(model.dtype), "width": p.width, "height": p.height }
|
||||
task_args = {
|
||||
'latents': init_latent.to(model.dtype),
|
||||
'width': p.width,
|
||||
'height': p.height,
|
||||
}
|
||||
return task_args
|
||||
|
||||
def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs):
|
||||
t0 = time.time()
|
||||
if hasattr(model, "set_progress_bar_config"):
|
||||
model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba')
|
||||
args = {}
|
||||
@@ -307,12 +340,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
pass
|
||||
task_kwargs = task_specific_kwargs(model)
|
||||
for arg in task_kwargs:
|
||||
if arg in possible and arg not in args: # task specific args should not override args
|
||||
# if arg in possible and arg not in args: # task specific args should not override args
|
||||
if arg in possible:
|
||||
args[arg] = task_kwargs[arg]
|
||||
else:
|
||||
pass
|
||||
# shared.log.debug(f'Diffuser not supported: pipeline={pipeline.__class__.__name__} task={sd_models.get_diffusers_task(model)} arg={arg}')
|
||||
# shared.log.debug(f'Diffuser pipeline: {model.__class__.__name__} possible={possible}')
|
||||
task_args = getattr(p, 'task_args', {})
|
||||
for k, v in task_args.items():
|
||||
args[k] = v
|
||||
|
||||
hypertile_set(p, hr=len(getattr(p, 'init_images', [])))
|
||||
clean = args.copy()
|
||||
clean.pop('callback', None)
|
||||
@@ -327,6 +361,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
clean['mask_image'] = type(clean['mask_image'])
|
||||
if 'masked_image_latents' in clean:
|
||||
clean['masked_image_latents'] = type(clean['masked_image_latents'])
|
||||
if 'ip_adapter_image' in clean:
|
||||
clean['ip_adapter_image'] = type(clean['ip_adapter_image'])
|
||||
if 'prompt' in clean:
|
||||
clean['prompt'] = len(clean['prompt'])
|
||||
if 'negative_prompt' in clean:
|
||||
@@ -342,8 +378,17 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
clean['generator'] = generator_device
|
||||
clean['parser'] = parser
|
||||
shared.log.debug(f'Diffuser pipeline: {model.__class__.__name__} task={sd_models.get_diffusers_task(model)} set={clean}')
|
||||
if p.hdr_clamp or p.hdr_center or p.hdr_maximize:
|
||||
txt = 'HDR:'
|
||||
txt += f' Clamp threshold={p.hdr_threshold} boundary={p.hdr_boundary}' if p.hdr_clamp else 'Clamp off'
|
||||
txt += f' Center channel-shift={p.hdr_channel_shift} full-shift={p.hdr_full_shift}' if p.hdr_center else 'Center off'
|
||||
txt += f' Maximize boundary={p.hdr_max_boundry} center={p.hdr_max_center}' if p.hdr_maximize else 'Maximize off'
|
||||
shared.log.debug(txt)
|
||||
# components = [{ k: getattr(v, 'device', None) } for k, v in model.components.items()]
|
||||
# shared.log.debug(f'Diffuser pipeline components: {components}')
|
||||
if shared.cmd_opts.profile:
|
||||
t1 = time.time()
|
||||
shared.log.debug(f'Profile: pipeline args: {t1-t0:.2f}')
|
||||
return args
|
||||
|
||||
def recompile_model(hires=False):
|
||||
@@ -375,8 +420,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
|
||||
def update_sampler(sd_model, second_pass=False):
|
||||
sampler_selection = p.latent_sampler if second_pass else p.sampler_name
|
||||
is_karras_compatible = sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers
|
||||
if hasattr(sd_model, 'scheduler') and sampler_selection != 'Default' and is_karras_compatible:
|
||||
# is_karras_compatible = sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers
|
||||
if hasattr(sd_model, 'scheduler') and sampler_selection != 'Default':
|
||||
sampler = sd_samplers.all_samplers_map.get(sampler_selection, None)
|
||||
if sampler is None:
|
||||
sampler = sd_samplers.all_samplers_map.get("UniPC")
|
||||
@@ -384,10 +429,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
# TODO extra_generation_params add sampler options
|
||||
# p.extra_generation_params['Sampler options'] = ''
|
||||
|
||||
recompile_model()
|
||||
update_sampler(shared.sd_model)
|
||||
p.extra_generation_params['Pipeline'] = shared.sd_model.__class__.__name__
|
||||
|
||||
if len(getattr(p, 'init_images', [])) > 0:
|
||||
while len(p.init_images) < len(prompts):
|
||||
p.init_images.append(p.init_images[-1])
|
||||
@@ -457,6 +498,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts,
|
||||
num_inference_steps=calculate_base_steps(),
|
||||
eta=shared.opts.scheduler_eta,
|
||||
guidance_scale=p.cfg_scale,
|
||||
guidance_rescale=p.diffusers_guidance_rescale,
|
||||
denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None,
|
||||
denoising_end=p.refiner_start if use_refiner_start else 1 if use_denoise_start else None,
|
||||
@@ -464,17 +506,31 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
clip_skip=p.clip_skip,
|
||||
desc='Base',
|
||||
)
|
||||
recompile_model()
|
||||
update_sampler(shared.sd_model)
|
||||
shared.state.sampling_steps = base_args['num_inference_steps']
|
||||
p.extra_generation_params['Pipeline'] = shared.sd_model.__class__.__name__
|
||||
p.extra_generation_params["Sampler Eta"] = shared.opts.scheduler_eta if shared.opts.scheduler_eta is not None and shared.opts.scheduler_eta > 0 and shared.opts.scheduler_eta < 1 else None
|
||||
try:
|
||||
t0 = time.time()
|
||||
output = shared.sd_model(**base_args) # pylint: disable=not-callable
|
||||
if shared.cmd_opts.profile:
|
||||
t1 = time.time()
|
||||
shared.log.debug(f'Profile: pipeline call: {t1-t0:.2f}')
|
||||
if not hasattr(output, 'images') and hasattr(output, 'frames'):
|
||||
shared.log.debug(f'Generated: frames={len(output.frames[0])}')
|
||||
output.images = output.frames[0]
|
||||
except AssertionError as e:
|
||||
shared.log.info(e)
|
||||
except ValueError as e:
|
||||
shared.state.interrupted = True
|
||||
shared.log.error(f'Processing: {e}')
|
||||
shared.log.error(f'Processing: args={base_args} {e}')
|
||||
if shared.cmd_opts.debug:
|
||||
errors.display(e, 'Processing')
|
||||
except RuntimeError as e:
|
||||
shared.state.interrupted = True
|
||||
shared.log.error(f'Processing: args={base_args} {e}')
|
||||
errors.display(e, 'Processing')
|
||||
|
||||
if hasattr(shared.sd_model, 'embedding_db') and len(shared.sd_model.embedding_db.embeddings_used) > 0:
|
||||
p.extra_generation_params['Embeddings'] = ', '.join(shared.sd_model.embedding_db.embeddings_used)
|
||||
|
||||
@@ -59,9 +59,9 @@ class DiffusersTextualInversionManager(BaseTextualInversionManager):
|
||||
return self.pipe.tokenizer.encode(prompt, add_special_tokens=False)
|
||||
|
||||
|
||||
def encode_prompts(pipeline, prompts: list, negative_prompts: list, clip_skip: typing.Optional[int] = None):
|
||||
if 'StableDiffusion' not in pipeline.__class__.__name__:
|
||||
shared.log.warning(f"Prompt parser not supported: {pipeline.__class__.__name__}")
|
||||
def encode_prompts(pipe, prompts: list, negative_prompts: list, clip_skip: typing.Optional[int] = None):
|
||||
if 'StableDiffusion' not in pipe.__class__.__name__:
|
||||
shared.log.warning(f"Prompt parser not supported: {pipe.__class__.__name__}")
|
||||
return None, None, None, None
|
||||
else:
|
||||
prompt_embeds = []
|
||||
@@ -69,7 +69,7 @@ def encode_prompts(pipeline, prompts: list, negative_prompts: list, clip_skip: t
|
||||
negative_embeds = []
|
||||
negative_pooleds = []
|
||||
for i in range(len(prompts)):
|
||||
prompt_embed, positive_pooled, negative_embed, negative_pooled = get_weighted_text_embeddings(pipeline, prompts[i], negative_prompts[i], clip_skip)
|
||||
prompt_embed, positive_pooled, negative_embed, negative_pooled = get_weighted_text_embeddings(pipe, prompts[i], negative_prompts[i], clip_skip)
|
||||
prompt_embeds.append(prompt_embed)
|
||||
positive_pooleds.append(positive_pooled)
|
||||
negative_embeds.append(negative_embed)
|
||||
@@ -118,9 +118,9 @@ def prepare_embedding_providers(pipe, clip_skip):
|
||||
def pad_to_same_length(pipe, embeds):
|
||||
device = pipe.device if str(pipe.device) != 'meta' else devices.device
|
||||
try: #SDXL
|
||||
empty_embed = shared.sd_model.encode_prompt("")
|
||||
empty_embed = pipe.encode_prompt("")
|
||||
except Exception: #SD1.5
|
||||
empty_embed = shared.sd_model.encode_prompt("", device, 1, False)
|
||||
empty_embed = pipe.encode_prompt("", device, 1, False)
|
||||
empty_batched = torch.cat([empty_embed[0].to(embeds[0].device)] * embeds[0].shape[0])
|
||||
max_token_count = max([embed.shape[1] for embed in embeds])
|
||||
for i, embed in enumerate(embeds):
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/bin/env python
|
||||
|
||||
import _thread
|
||||
import os
|
||||
import time
|
||||
from queue import Queue
|
||||
import cv2
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch.nn import functional as F
|
||||
from tqdm.rich import tqdm
|
||||
from modules.rife.ssim import ssim_matlab
|
||||
from modules.rife.model_rife import Model
|
||||
from modules import devices, shared
|
||||
|
||||
|
||||
model_url = 'https://github.com/vladmandic/rife/raw/main/model/flownet-v46.pkl'
|
||||
model = None
|
||||
|
||||
|
||||
def load(model_path: str = 'rife/flownet-v46.pkl'):
|
||||
global model # pylint: disable=global-statement
|
||||
if model is None:
|
||||
from modules import modelloader
|
||||
model_dir = os.path.join(shared.models_path, 'RIFE')
|
||||
model_path = modelloader.load_file_from_url(url=model_url, model_dir=model_dir, file_name='flownet-v46.pkl')
|
||||
shared.log.debug(f'RIFE load model: file="{model_path}"')
|
||||
model = Model()
|
||||
model.load_model(model_path, -1)
|
||||
model.eval()
|
||||
model.device()
|
||||
|
||||
|
||||
def interpolate(images: list, count: int = 2, scale: float = 1.0, pad: int = 1, change: float = 0.3):
|
||||
if images is None or len(images) < 2:
|
||||
return []
|
||||
if model is None:
|
||||
load()
|
||||
interpolated = []
|
||||
h = images[0].height
|
||||
w = images[0].width
|
||||
t0 = time.time()
|
||||
|
||||
def write(buffer):
|
||||
item = buffer.get()
|
||||
while item is not None:
|
||||
img = item[:, :, ::-1]
|
||||
# image = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
|
||||
image = Image.fromarray(img)
|
||||
item = buffer.get()
|
||||
interpolated.append(image)
|
||||
|
||||
def execute(I0, I1, n):
|
||||
if model.version >= 3.9:
|
||||
res = []
|
||||
for i in range(n):
|
||||
res.append(model.inference(I0, I1, (i+1) * 1. / (n+1), scale))
|
||||
return res
|
||||
else:
|
||||
middle = model.inference(I0, I1, scale)
|
||||
if n == 1:
|
||||
return [middle]
|
||||
first_half = execute(I0, middle, n=n//2)
|
||||
second_half = execute(middle, I1, n=n//2)
|
||||
if n % 2:
|
||||
return [*first_half, middle, *second_half]
|
||||
else:
|
||||
return [*first_half, *second_half]
|
||||
|
||||
def f_pad(img):
|
||||
return F.pad(img, padding).to(devices.dtype) # pylint: disable=not-callable
|
||||
|
||||
tmp = max(128, int(128 / scale))
|
||||
ph = ((h - 1) // tmp + 1) * tmp
|
||||
pw = ((w - 1) // tmp + 1) * tmp
|
||||
padding = (0, pw - w, 0, ph - h)
|
||||
buffer = Queue(maxsize=8192)
|
||||
_thread.start_new_thread(write, (buffer,))
|
||||
|
||||
frame = cv2.cvtColor(np.array(images[0]), cv2.COLOR_RGB2BGR)
|
||||
for _i in range(pad): # fill starting frames
|
||||
buffer.put(frame)
|
||||
|
||||
I1 = f_pad(torch.from_numpy(np.transpose(frame, (2,0,1))).to(devices.device, non_blocking=True).unsqueeze(0).float() / 255.)
|
||||
with torch.no_grad():
|
||||
with tqdm(total=len(images), desc='Interpolate', unit='frame') as pbar:
|
||||
for image in images:
|
||||
frame = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
|
||||
I0 = I1
|
||||
I1 = f_pad(torch.from_numpy(np.transpose(frame, (2,0,1))).to(devices.device, non_blocking=True).unsqueeze(0).float() / 255.)
|
||||
I0_small = F.interpolate(I0, (32, 32), mode='bilinear', align_corners=False).to(torch.float32)
|
||||
I1_small = F.interpolate(I1, (32, 32), mode='bilinear', align_corners=False).to(torch.float32)
|
||||
ssim = ssim_matlab(I0_small[:, :3], I1_small[:, :3])
|
||||
if ssim > 0.99: # skip duplicate frames
|
||||
continue
|
||||
if ssim < change:
|
||||
output = []
|
||||
for _i in range(pad): # fill frames if change rate is above threshold
|
||||
output.append(I0)
|
||||
for _i in range(pad):
|
||||
output.append(I1)
|
||||
else:
|
||||
output = execute(I0, I1, count-1)
|
||||
for mid in output:
|
||||
mid = (((mid[0] * 255.).byte().cpu().numpy().transpose(1, 2, 0)))
|
||||
buffer.put(mid[:h, :w])
|
||||
buffer.put(frame)
|
||||
pbar.update(1)
|
||||
|
||||
for _i in range(pad): # fill ending frames
|
||||
buffer.put(frame)
|
||||
while not buffer.empty():
|
||||
time.sleep(0.1)
|
||||
t1 = time.time()
|
||||
shared.log.info(f'RIFE interpolate: input={len(images)} frames={len(interpolated)} resolution={w}x{h} interpolate={count} scale={scale} pad={pad} change={change} time={round(t1 - t0, 2)}')
|
||||
return interpolated
|
||||
@@ -0,0 +1,122 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torchvision.models as models
|
||||
from modules import devices
|
||||
|
||||
|
||||
class EPE(nn.Module):
|
||||
def __init__(self):
|
||||
super(EPE, self).__init__()
|
||||
|
||||
def forward(self, flow, gt, loss_mask):
|
||||
loss_map = (flow - gt.detach()) ** 2
|
||||
loss_map = (loss_map.sum(1, True) + 1e-6) ** 0.5
|
||||
return loss_map * loss_mask
|
||||
|
||||
|
||||
class Ternary(nn.Module):
|
||||
def __init__(self):
|
||||
super(Ternary, self).__init__()
|
||||
patch_size = 7
|
||||
out_channels = patch_size * patch_size
|
||||
self.w = np.eye(out_channels).reshape(
|
||||
(patch_size, patch_size, 1, out_channels))
|
||||
self.w = np.transpose(self.w, (3, 2, 0, 1))
|
||||
self.w = torch.tensor(self.w).float().to(devices.device)
|
||||
|
||||
def transform(self, img):
|
||||
patches = F.conv2d(img, self.w, padding=3, bias=None)
|
||||
transf = patches - img
|
||||
transf_norm = transf / torch.sqrt(0.81 + transf**2)
|
||||
return transf_norm
|
||||
|
||||
def rgb2gray(self, rgb):
|
||||
r, g, b = rgb[:, 0:1, :, :], rgb[:, 1:2, :, :], rgb[:, 2:3, :, :]
|
||||
gray = 0.2989 * r + 0.5870 * g + 0.1140 * b
|
||||
return gray
|
||||
|
||||
def hamming(self, t1, t2):
|
||||
dist = (t1 - t2) ** 2
|
||||
dist_norm = torch.mean(dist / (0.1 + dist), 1, True)
|
||||
return dist_norm
|
||||
|
||||
def valid_mask(self, t, padding):
|
||||
n, _, h, w = t.size()
|
||||
inner = torch.ones(n, 1, h - 2 * padding, w - 2 * padding).type_as(t)
|
||||
mask = F.pad(inner, [padding] * 4)
|
||||
return mask
|
||||
|
||||
def forward(self, img0, img1):
|
||||
img0 = self.transform(self.rgb2gray(img0))
|
||||
img1 = self.transform(self.rgb2gray(img1))
|
||||
return self.hamming(img0, img1) * self.valid_mask(img0, 1)
|
||||
|
||||
|
||||
class SOBEL(nn.Module):
|
||||
def __init__(self):
|
||||
super(SOBEL, self).__init__()
|
||||
self.kernelX = torch.tensor([
|
||||
[1, 0, -1],
|
||||
[2, 0, -2],
|
||||
[1, 0, -1],
|
||||
]).float()
|
||||
self.kernelY = self.kernelX.clone().T
|
||||
self.kernelX = self.kernelX.unsqueeze(0).unsqueeze(0).to(devices.device)
|
||||
self.kernelY = self.kernelY.unsqueeze(0).unsqueeze(0).to(devices.device)
|
||||
|
||||
def forward(self, pred, gt):
|
||||
N, C, H, W = pred.shape[0], pred.shape[1], pred.shape[2], pred.shape[3]
|
||||
img_stack = torch.cat(
|
||||
[pred.reshape(N*C, 1, H, W), gt.reshape(N*C, 1, H, W)], 0)
|
||||
sobel_stack_x = F.conv2d(img_stack, self.kernelX, padding=1)
|
||||
sobel_stack_y = F.conv2d(img_stack, self.kernelY, padding=1)
|
||||
pred_X, gt_X = sobel_stack_x[:N*C], sobel_stack_x[N*C:]
|
||||
pred_Y, gt_Y = sobel_stack_y[:N*C], sobel_stack_y[N*C:]
|
||||
L1X, L1Y = torch.abs(pred_X-gt_X), torch.abs(pred_Y-gt_Y)
|
||||
loss = L1X+L1Y
|
||||
return loss
|
||||
|
||||
|
||||
class MeanShift(nn.Conv2d):
|
||||
def __init__(self, data_mean, data_std, data_range=1, norm=True):
|
||||
c = len(data_mean)
|
||||
super(MeanShift, self).__init__(c, c, kernel_size=1)
|
||||
std = torch.Tensor(data_std)
|
||||
self.weight.data = torch.eye(c).view(c, c, 1, 1)
|
||||
if norm:
|
||||
self.weight.data.div_(std.view(c, 1, 1, 1))
|
||||
self.bias.data = -1 * data_range * torch.Tensor(data_mean)
|
||||
self.bias.data.div_(std)
|
||||
else:
|
||||
self.weight.data.mul_(std.view(c, 1, 1, 1))
|
||||
self.bias.data = data_range * torch.Tensor(data_mean)
|
||||
self.requires_grad = False
|
||||
|
||||
|
||||
class VGGPerceptualLoss(torch.nn.Module):
|
||||
def __init__(self, rank=0): # pylint: disable=unused-argument
|
||||
super(VGGPerceptualLoss, self).__init__()
|
||||
pretrained = True
|
||||
self.vgg_pretrained_features = models.vgg19(
|
||||
pretrained=pretrained).features
|
||||
self.normalize = MeanShift([0.485, 0.456, 0.406], [
|
||||
0.229, 0.224, 0.225], norm=True).cuda()
|
||||
for param in self.parameters():
|
||||
param.requires_grad = False
|
||||
|
||||
def forward(self, X, Y, indices=None):
|
||||
X = self.normalize(X)
|
||||
Y = self.normalize(Y)
|
||||
indices = [2, 7, 12, 21, 30]
|
||||
weights = [1.0/2.6, 1.0/4.8, 1.0/3.7, 1.0/5.6, 10/1.5]
|
||||
k = 0
|
||||
loss = 0
|
||||
for i in range(indices[-1]):
|
||||
X = self.vgg_pretrained_features[i](X)
|
||||
Y = self.vgg_pretrained_features[i](Y)
|
||||
if i+1 in indices:
|
||||
loss += weights[k] * (X - Y.detach()).abs().mean() * 0.1
|
||||
k += 1
|
||||
return loss
|
||||
@@ -0,0 +1,134 @@
|
||||
import os
|
||||
import sys
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
sys.path.append(os.path.dirname(__file__))
|
||||
from warplayer import warp # pylint: disable=wrong-import-position
|
||||
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
|
||||
def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
|
||||
return nn.Sequential(
|
||||
nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride,
|
||||
padding=padding, dilation=dilation, bias=True),
|
||||
nn.LeakyReLU(0.2, True)
|
||||
)
|
||||
|
||||
def conv_bn(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
|
||||
return nn.Sequential(
|
||||
nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride,
|
||||
padding=padding, dilation=dilation, bias=False),
|
||||
nn.BatchNorm2d(out_planes),
|
||||
nn.LeakyReLU(0.2, True)
|
||||
)
|
||||
|
||||
class ResConv(nn.Module):
|
||||
def __init__(self, c, dilation=1):
|
||||
super(ResConv, self).__init__()
|
||||
self.conv = nn.Conv2d(c, c, 3, 1, dilation, dilation=dilation, groups=1\
|
||||
)
|
||||
self.beta = nn.Parameter(torch.ones((1, c, 1, 1)), requires_grad=True)
|
||||
self.relu = nn.LeakyReLU(0.2, True)
|
||||
|
||||
def forward(self, x):
|
||||
return self.relu(self.conv(x) * self.beta + x)
|
||||
|
||||
class IFBlock(nn.Module):
|
||||
def __init__(self, in_planes, c=64):
|
||||
super(IFBlock, self).__init__()
|
||||
self.conv0 = nn.Sequential(
|
||||
conv(in_planes, c//2, 3, 2, 1),
|
||||
conv(c//2, c, 3, 2, 1),
|
||||
)
|
||||
self.convblock = nn.Sequential(
|
||||
ResConv(c),
|
||||
ResConv(c),
|
||||
ResConv(c),
|
||||
ResConv(c),
|
||||
ResConv(c),
|
||||
ResConv(c),
|
||||
ResConv(c),
|
||||
ResConv(c),
|
||||
)
|
||||
self.lastconv = nn.Sequential(
|
||||
nn.ConvTranspose2d(c, 4*6, 4, 2, 1),
|
||||
nn.PixelShuffle(2)
|
||||
)
|
||||
|
||||
def forward(self, x, flow=None, scale=1):
|
||||
x = F.interpolate(x, scale_factor= 1. / scale, mode="bilinear", align_corners=False)
|
||||
if flow is not None:
|
||||
flow = F.interpolate(flow, scale_factor= 1. / scale, mode="bilinear", align_corners=False) * 1. / scale
|
||||
x = torch.cat((x, flow), 1)
|
||||
feat = self.conv0(x)
|
||||
feat = self.convblock(feat)
|
||||
tmp = self.lastconv(feat)
|
||||
tmp = F.interpolate(tmp, scale_factor=scale, mode="bilinear", align_corners=False)
|
||||
flow = tmp[:, :4] * scale
|
||||
mask = tmp[:, 4:5]
|
||||
return flow, mask
|
||||
|
||||
class IFNet(nn.Module):
|
||||
def __init__(self):
|
||||
super(IFNet, self).__init__()
|
||||
self.block0 = IFBlock(7, c=192)
|
||||
self.block1 = IFBlock(8+4, c=128)
|
||||
self.block2 = IFBlock(8+4, c=96)
|
||||
self.block3 = IFBlock(8+4, c=64)
|
||||
# self.contextnet = Contextnet()
|
||||
# self.unet = Unet()
|
||||
|
||||
def forward( self, x, timestep=0.5, scale_list=[8, 4, 2, 1], training=False, fastmode=True, ensemble=False): # pylint: disable=dangerous-default-value # noqa: B006
|
||||
if training is False:
|
||||
channel = x.shape[1] // 2
|
||||
img0 = x[:, :channel]
|
||||
img1 = x[:, channel:]
|
||||
if not torch.is_tensor(timestep):
|
||||
timestep = (x[:, :1].clone() * 0 + 1) * timestep
|
||||
else:
|
||||
timestep = timestep.repeat(1, 1, img0.shape[2], img0.shape[3])
|
||||
flow_list = []
|
||||
merged = []
|
||||
mask_list = []
|
||||
warped_img0 = img0
|
||||
warped_img1 = img1
|
||||
flow = None
|
||||
mask = None
|
||||
# loss_cons = 0
|
||||
block = [self.block0, self.block1, self.block2, self.block3]
|
||||
for i in range(4):
|
||||
if flow is None:
|
||||
flow, mask = block[i](torch.cat((img0[:, :3], img1[:, :3], timestep), 1), None, scale=scale_list[i])
|
||||
if ensemble:
|
||||
f1, m1 = block[i](torch.cat((img1[:, :3], img0[:, :3], 1-timestep), 1), None, scale=scale_list[i])
|
||||
flow = (flow + torch.cat((f1[:, 2:4], f1[:, :2]), 1)) / 2
|
||||
mask = (mask + (-m1)) / 2
|
||||
else:
|
||||
f0, m0 = block[i](torch.cat((warped_img0[:, :3], warped_img1[:, :3], timestep, mask), 1), flow, scale=scale_list[i])
|
||||
if ensemble:
|
||||
f1, m1 = block[i](torch.cat((warped_img1[:, :3], warped_img0[:, :3], 1-timestep, -mask), 1), torch.cat((flow[:, 2:4], flow[:, :2]), 1), scale=scale_list[i]) # pylint: disable=invalid-unary-operand-type
|
||||
f0 = (f0 + torch.cat((f1[:, 2:4], f1[:, :2]), 1)) / 2
|
||||
m0 = (m0 + (-m1)) / 2
|
||||
flow = flow + f0
|
||||
mask = mask + m0
|
||||
mask_list.append(mask)
|
||||
flow_list.append(flow)
|
||||
warped_img0 = warp(img0, flow[:, :2])
|
||||
warped_img1 = warp(img1, flow[:, 2:4])
|
||||
merged.append((warped_img0, warped_img1))
|
||||
mask_list[3] = torch.sigmoid(mask_list[3])
|
||||
merged[3] = merged[3][0] * mask_list[3] + merged[3][1] * (1 - mask_list[3])
|
||||
if not fastmode:
|
||||
print('contextnet is removed')
|
||||
'''
|
||||
c0 = self.contextnet(img0, flow[:, :2])
|
||||
c1 = self.contextnet(img1, flow[:, 2:4])
|
||||
tmp = self.unet(img0, img1, warped_img0, warped_img1, mask, flow, c0, c1)
|
||||
res = tmp[:, :3] * 2 - 1
|
||||
merged[3] = torch.clamp(merged[3] + res, 0, 1)
|
||||
'''
|
||||
return flow_list, mask_list[3], merged
|
||||
@@ -0,0 +1,79 @@
|
||||
import torch
|
||||
from torch.optim import AdamW
|
||||
from torch.nn.parallel import DistributedDataParallel as DDP
|
||||
from modules.rife.model_ifnet import IFNet
|
||||
from modules.rife.loss import EPE, SOBEL
|
||||
from modules import devices
|
||||
|
||||
|
||||
class Model:
|
||||
def __init__(self, local_rank=-1):
|
||||
self.flownet = IFNet()
|
||||
self.device()
|
||||
self.optimG = AdamW(self.flownet.parameters(), lr=1e-6, weight_decay=1e-4)
|
||||
self.epe = EPE()
|
||||
self.version = 3.9
|
||||
# self.vgg = VGGPerceptualLoss().to(device)
|
||||
self.sobel = SOBEL()
|
||||
if local_rank != -1:
|
||||
self.flownet = DDP(self.flownet, device_ids=[local_rank], output_device=local_rank)
|
||||
|
||||
def train(self):
|
||||
self.flownet.train()
|
||||
|
||||
def eval(self):
|
||||
self.flownet.eval()
|
||||
|
||||
def device(self):
|
||||
self.flownet.to(devices.device)
|
||||
self.flownet.to(devices.dtype)
|
||||
|
||||
def load_model(self, model_file, rank=0):
|
||||
def convert(param):
|
||||
if rank == -1:
|
||||
return { k.replace("module.", ""): v for k, v in param.items() if "module." in k }
|
||||
else:
|
||||
return param
|
||||
if rank <= 0:
|
||||
if torch.cuda.is_available():
|
||||
self.flownet.load_state_dict(convert(torch.load(model_file)), False)
|
||||
else:
|
||||
self.flownet.load_state_dict(convert(torch.load(model_file, map_location='cpu')), False)
|
||||
|
||||
def save_model(self, model_file, rank=0):
|
||||
if rank == 0:
|
||||
torch.save(self.flownet.state_dict(), model_file)
|
||||
|
||||
def inference(self, img0, img1, timestep=0.5, scale=1.0):
|
||||
imgs = torch.cat((img0, img1), 1)
|
||||
scale_list = [8/scale, 4/scale, 2/scale, 1/scale]
|
||||
_flow, _mask, merged = self.flownet(imgs, timestep, scale_list)
|
||||
return merged[3]
|
||||
|
||||
def update(self, imgs, gt, learning_rate=0, mul=1, training=True, flow_gt=None): # pylint: disable=unused-argument
|
||||
for param_group in self.optimG.param_groups:
|
||||
param_group['lr'] = learning_rate
|
||||
# img0 = imgs[:, :3]
|
||||
# img1 = imgs[:, 3:]
|
||||
if training:
|
||||
self.train()
|
||||
else:
|
||||
self.eval()
|
||||
scale = [8, 4, 2, 1]
|
||||
flow, mask, merged = self.flownet(torch.cat((imgs, gt), 1), scale=scale, training=training)
|
||||
loss_l1 = (merged[3] - gt).abs().mean()
|
||||
loss_smooth = self.sobel(flow[3], flow[3]*0).mean()
|
||||
# loss_vgg = self.vgg(merged[2], gt)
|
||||
if training:
|
||||
self.optimG.zero_grad()
|
||||
loss_G = loss_l1 + loss_smooth * 0.1
|
||||
loss_G.backward()
|
||||
self.optimG.step()
|
||||
# else:
|
||||
# flow_teacher = flow[2]
|
||||
return merged[3], {
|
||||
'mask': mask,
|
||||
'flow': flow[3][:, :2],
|
||||
'loss_l1': loss_l1,
|
||||
'loss_smooth': loss_smooth,
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from modules.rife.warplayer import warp
|
||||
|
||||
|
||||
c = 16
|
||||
|
||||
|
||||
def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
|
||||
return nn.Sequential(
|
||||
nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=True),
|
||||
nn.LeakyReLU(0.2, True)
|
||||
)
|
||||
|
||||
|
||||
def conv_woact(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
|
||||
return nn.Sequential(
|
||||
nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=True),
|
||||
)
|
||||
|
||||
|
||||
def deconv(in_planes, out_planes, kernel_size=4, stride=2, padding=1): # pylint: disable=unused-argument
|
||||
return nn.Sequential(
|
||||
torch.nn.ConvTranspose2d(in_channels=in_planes, out_channels=out_planes, kernel_size=4, stride=2, padding=1, bias=True),
|
||||
nn.LeakyReLU(0.2, True)
|
||||
)
|
||||
|
||||
|
||||
class Conv2(nn.Module):
|
||||
def __init__(self, in_planes, out_planes, stride=2):
|
||||
super(Conv2, self).__init__()
|
||||
self.conv1 = conv(in_planes, out_planes, 3, stride, 1)
|
||||
self.conv2 = conv(out_planes, out_planes, 3, 1, 1)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.conv1(x)
|
||||
x = self.conv2(x)
|
||||
return x
|
||||
|
||||
|
||||
class Contextnet(nn.Module):
|
||||
def __init__(self):
|
||||
super(Contextnet, self).__init__()
|
||||
self.conv1 = Conv2(3, c)
|
||||
self.conv2 = Conv2(c, 2*c)
|
||||
self.conv3 = Conv2(2*c, 4*c)
|
||||
self.conv4 = Conv2(4*c, 8*c)
|
||||
|
||||
def forward(self, x, flow):
|
||||
x = self.conv1(x)
|
||||
flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False) * 0.5
|
||||
f1 = warp(x, flow)
|
||||
x = self.conv2(x)
|
||||
flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False) * 0.5
|
||||
f2 = warp(x, flow)
|
||||
x = self.conv3(x)
|
||||
flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False) * 0.5
|
||||
f3 = warp(x, flow)
|
||||
x = self.conv4(x)
|
||||
flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False) * 0.5
|
||||
f4 = warp(x, flow)
|
||||
return [f1, f2, f3, f4]
|
||||
|
||||
|
||||
class Unet(nn.Module):
|
||||
def __init__(self):
|
||||
super(Unet, self).__init__()
|
||||
self.down0 = Conv2(17, 2*c)
|
||||
self.down1 = Conv2(4*c, 4*c)
|
||||
self.down2 = Conv2(8*c, 8*c)
|
||||
self.down3 = Conv2(16*c, 16*c)
|
||||
self.up0 = deconv(32*c, 8*c)
|
||||
self.up1 = deconv(16*c, 4*c)
|
||||
self.up2 = deconv(8*c, 2*c)
|
||||
self.up3 = deconv(4*c, c)
|
||||
self.conv = nn.Conv2d(c, 3, 3, 1, 1)
|
||||
|
||||
def forward(self, img0, img1, warped_img0, warped_img1, mask, flow, c0, c1):
|
||||
s0 = self.down0(
|
||||
torch.cat((img0, img1, warped_img0, warped_img1, mask, flow), 1))
|
||||
s1 = self.down1(torch.cat((s0, c0[0], c1[0]), 1))
|
||||
s2 = self.down2(torch.cat((s1, c0[1], c1[1]), 1))
|
||||
s3 = self.down3(torch.cat((s2, c0[2], c1[2]), 1))
|
||||
x = self.up0(torch.cat((s3, c0[3], c1[3]), 1))
|
||||
x = self.up1(torch.cat((x, s2), 1))
|
||||
x = self.up2(torch.cat((x, s1), 1))
|
||||
x = self.up3(torch.cat((x, s0), 1))
|
||||
x = self.conv(x)
|
||||
return torch.sigmoid(x)
|
||||
@@ -0,0 +1,174 @@
|
||||
from math import exp
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from modules import devices
|
||||
|
||||
|
||||
def gaussian(window_size, sigma):
|
||||
gauss = torch.Tensor([exp(-(x - window_size//2)**2/float(2*sigma**2)) for x in range(window_size)])
|
||||
return gauss/gauss.sum()
|
||||
|
||||
|
||||
def create_window(window_size, channel=1):
|
||||
_1D_window = gaussian(window_size, 1.5).unsqueeze(1)
|
||||
_2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0).to(devices.device)
|
||||
window = _2D_window.expand(channel, 1, window_size, window_size).contiguous()
|
||||
return window
|
||||
|
||||
|
||||
def create_window_3d(window_size, channel=1):
|
||||
_1D_window = gaussian(window_size, 1.5).unsqueeze(1)
|
||||
_2D_window = _1D_window.mm(_1D_window.t())
|
||||
_3D_window = _2D_window.unsqueeze(2) @ (_1D_window.t())
|
||||
window = _3D_window.expand(1, channel, window_size, window_size, window_size).contiguous().to(devices.device)
|
||||
return window
|
||||
|
||||
|
||||
def ssim(img1, img2, window_size=11, window=None, size_average=True, full=False, val_range=None):
|
||||
# Value range can be different from 255. Other common ranges are 1 (sigmoid) and 2 (tanh).
|
||||
if val_range is None:
|
||||
if torch.max(img1) > 128:
|
||||
max_val = 255
|
||||
else:
|
||||
max_val = 1
|
||||
|
||||
if torch.min(img1) < -0.5:
|
||||
min_val = -1
|
||||
else:
|
||||
min_val = 0
|
||||
L = max_val - min_val
|
||||
else:
|
||||
L = val_range
|
||||
padd = 0
|
||||
(_, channel, height, width) = img1.size()
|
||||
if window is None:
|
||||
real_size = min(window_size, height, width)
|
||||
window = create_window(real_size, channel=channel).to(img1.device)
|
||||
# mu1 = F.conv2d(img1, window, padding=padd, groups=channel)
|
||||
# mu2 = F.conv2d(img2, window, padding=padd, groups=channel)
|
||||
mu1 = F.conv2d(F.pad(img1, (5, 5, 5, 5), mode='replicate'), window, padding=padd, groups=channel)
|
||||
mu2 = F.conv2d(F.pad(img2, (5, 5, 5, 5), mode='replicate'), window, padding=padd, groups=channel)
|
||||
mu1_sq = mu1.pow(2)
|
||||
mu2_sq = mu2.pow(2)
|
||||
mu1_mu2 = mu1 * mu2
|
||||
sigma1_sq = F.conv2d(F.pad(img1 * img1, (5, 5, 5, 5), 'replicate'), window, padding=padd, groups=channel) - mu1_sq
|
||||
sigma2_sq = F.conv2d(F.pad(img2 * img2, (5, 5, 5, 5), 'replicate'), window, padding=padd, groups=channel) - mu2_sq
|
||||
sigma12 = F.conv2d(F.pad(img1 * img2, (5, 5, 5, 5), 'replicate'), window, padding=padd, groups=channel) - mu1_mu2
|
||||
C1 = (0.01 * L) ** 2
|
||||
C2 = (0.03 * L) ** 2
|
||||
v1 = 2.0 * sigma12 + C2
|
||||
v2 = sigma1_sq + sigma2_sq + C2
|
||||
cs = torch.mean(v1 / v2) # contrast sensitivity
|
||||
ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2)
|
||||
if size_average:
|
||||
ret = ssim_map.mean()
|
||||
else:
|
||||
ret = ssim_map.mean(1).mean(1).mean(1)
|
||||
if full:
|
||||
return ret, cs
|
||||
return ret
|
||||
|
||||
|
||||
def ssim_matlab(img1, img2, window_size=11, window=None, size_average=True, full=False, val_range=None):
|
||||
# Value range can be different from 255. Other common ranges are 1 (sigmoid) and 2 (tanh).
|
||||
if val_range is None:
|
||||
if torch.max(img1) > 128:
|
||||
max_val = 255
|
||||
else:
|
||||
max_val = 1
|
||||
if torch.min(img1) < -0.5:
|
||||
min_val = -1
|
||||
else:
|
||||
min_val = 0
|
||||
L = max_val - min_val
|
||||
else:
|
||||
L = val_range
|
||||
padd = 0
|
||||
(_, _, height, width) = img1.size()
|
||||
if window is None:
|
||||
real_size = min(window_size, height, width)
|
||||
window = create_window_3d(real_size, channel=1).to(img1.device)
|
||||
# Channel is set to 1 since we consider color images as volumetric images
|
||||
img1 = img1.unsqueeze(1)
|
||||
img2 = img2.unsqueeze(1)
|
||||
mu1 = F.conv3d(F.pad(img1, (5, 5, 5, 5, 5, 5), mode='replicate'), window, padding=padd, groups=1)
|
||||
mu2 = F.conv3d(F.pad(img2, (5, 5, 5, 5, 5, 5), mode='replicate'), window, padding=padd, groups=1)
|
||||
mu1_sq = mu1.pow(2)
|
||||
mu2_sq = mu2.pow(2)
|
||||
mu1_mu2 = mu1 * mu2
|
||||
sigma1_sq = F.conv3d(F.pad(img1 * img1, (5, 5, 5, 5, 5, 5), 'replicate'), window, padding=padd, groups=1) - mu1_sq
|
||||
sigma2_sq = F.conv3d(F.pad(img2 * img2, (5, 5, 5, 5, 5, 5), 'replicate'), window, padding=padd, groups=1) - mu2_sq
|
||||
sigma12 = F.conv3d(F.pad(img1 * img2, (5, 5, 5, 5, 5, 5), 'replicate'), window, padding=padd, groups=1) - mu1_mu2
|
||||
C1 = (0.01 * L) ** 2
|
||||
C2 = (0.03 * L) ** 2
|
||||
v1 = 2.0 * sigma12 + C2
|
||||
v2 = sigma1_sq + sigma2_sq + C2
|
||||
cs = torch.mean(v1 / v2) # contrast sensitivity
|
||||
ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2)
|
||||
if size_average:
|
||||
ret = ssim_map.mean()
|
||||
else:
|
||||
ret = ssim_map.mean(1).mean(1).mean(1)
|
||||
if full:
|
||||
return ret, cs
|
||||
return ret
|
||||
|
||||
|
||||
def msssim(img1, img2, window_size=11, size_average=True, val_range=None, normalize=False):
|
||||
local_device = img1.device
|
||||
weights = torch.FloatTensor([0.0448, 0.2856, 0.3001, 0.2363, 0.1333]).to(local_device)
|
||||
levels = weights.size()[0]
|
||||
mssim = []
|
||||
mcs = []
|
||||
for _ in range(levels):
|
||||
sim, cs = ssim(img1, img2, window_size=window_size, size_average=size_average, full=True, val_range=val_range)
|
||||
mssim.append(sim)
|
||||
mcs.append(cs)
|
||||
img1 = F.avg_pool2d(img1, (2, 2))
|
||||
img2 = F.avg_pool2d(img2, (2, 2))
|
||||
mssim = torch.stack(mssim)
|
||||
mcs = torch.stack(mcs)
|
||||
# Normalize (to avoid NaNs during training unstable models, not compliant with original definition)
|
||||
if normalize:
|
||||
mssim = (mssim + 1) / 2
|
||||
mcs = (mcs + 1) / 2
|
||||
pow1 = mcs ** weights
|
||||
pow2 = mssim ** weights
|
||||
# From Matlab implementation https://ece.uwaterloo.ca/~z70wang/research/iwssim/
|
||||
output = torch.prod(pow1[:-1] * pow2[-1])
|
||||
return output
|
||||
|
||||
|
||||
# Classes to re-use window
|
||||
class SSIM(torch.nn.Module):
|
||||
def __init__(self, window_size=11, size_average=True, val_range=None):
|
||||
super(SSIM, self).__init__()
|
||||
self.window_size = window_size
|
||||
self.size_average = size_average
|
||||
self.val_range = val_range
|
||||
# Assume 3 channel for SSIM
|
||||
self.channel = 3
|
||||
self.window = create_window(window_size, channel=self.channel)
|
||||
|
||||
def forward(self, img1, img2):
|
||||
(_, channel, _, _) = img1.size()
|
||||
if channel == self.channel and self.window.dtype == img1.dtype:
|
||||
window = self.window
|
||||
else:
|
||||
window = create_window(self.window_size, channel).to(img1.device).type(img1.dtype)
|
||||
self.window = window
|
||||
self.channel = channel
|
||||
_ssim = ssim(img1, img2, window=window, window_size=self.window_size, size_average=self.size_average)
|
||||
dssim = (1 - _ssim) / 2
|
||||
return dssim
|
||||
|
||||
|
||||
class MSSSIM(torch.nn.Module):
|
||||
def __init__(self, window_size=11, size_average=True, channel=3):
|
||||
super(MSSSIM, self).__init__()
|
||||
self.window_size = window_size
|
||||
self.size_average = size_average
|
||||
self.channel = channel
|
||||
|
||||
def forward(self, img1, img2):
|
||||
return msssim(img1, img2, window_size=self.window_size, size_average=self.size_average)
|
||||
@@ -0,0 +1,17 @@
|
||||
import torch
|
||||
from modules import devices
|
||||
|
||||
|
||||
backwarp_tenGrid = {}
|
||||
|
||||
|
||||
def warp(tenInput, tenFlow):
|
||||
k = (str(tenFlow.device), str(tenFlow.size()))
|
||||
if k not in backwarp_tenGrid:
|
||||
tenHorizontal = torch.linspace(-1.0, 1.0, tenFlow.shape[3], device=devices.device).view(1, 1, 1, tenFlow.shape[3]).expand(tenFlow.shape[0], -1, tenFlow.shape[2], -1)
|
||||
tenVertical = torch.linspace(-1.0, 1.0, tenFlow.shape[2], device=devices.device).view(1, 1, tenFlow.shape[2], 1).expand(tenFlow.shape[0], -1, -1, tenFlow.shape[3])
|
||||
backwarp_tenGrid[k] = torch.cat([tenHorizontal, tenVertical], 1).to(devices.device)
|
||||
tenFlow = torch.cat([tenFlow[:, 0:1, :, :] / ((tenInput.shape[3] - 1.0) / 2.0),
|
||||
tenFlow[:, 1:2, :, :] / ((tenInput.shape[2] - 1.0) / 2.0)], 1)
|
||||
grid = (backwarp_tenGrid[k] + tenFlow).permute(0, 2, 3, 1).to(devices.dtype)
|
||||
return torch.nn.functional.grid_sample(input=tenInput, grid=grid, mode='bilinear', padding_mode='border', align_corners=True)
|
||||
@@ -329,7 +329,7 @@ def before_ui_callback():
|
||||
|
||||
|
||||
def add_callback(callbacks, fun):
|
||||
stack = [x for x in inspect.stack() if x.filename != __file__]
|
||||
stack = [x for x in inspect.stack(0) if x.filename != __file__]
|
||||
filename = stack[0].filename if len(stack) > 0 else 'unknown file'
|
||||
callbacks.append(ScriptCallback(filename, fun))
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import modules.errors as errors
|
||||
from installer import setup_logging
|
||||
from installer import setup_logging, args
|
||||
|
||||
|
||||
preloaded = []
|
||||
@@ -12,6 +12,10 @@ preloaded = []
|
||||
def load_module(path):
|
||||
module_spec = importlib.util.spec_from_file_location(os.path.basename(path), path)
|
||||
module = importlib.util.module_from_spec(module_spec)
|
||||
if args.profile:
|
||||
import cProfile
|
||||
pr = cProfile.Profile()
|
||||
pr.enable()
|
||||
try:
|
||||
if '/sd-extension-' in path: # safe extensions without stdout intercept
|
||||
module_spec.loader.exec_module(module)
|
||||
@@ -25,6 +29,8 @@ def load_module(path):
|
||||
errors.log.info(f"Extension: script='{os.path.relpath(path)}' {line.strip()}")
|
||||
except Exception as e:
|
||||
errors.display(e, f'Module load: {path}')
|
||||
if args.profile:
|
||||
errors.profile(pr, f'Scripts: {path}')
|
||||
return module
|
||||
|
||||
|
||||
|
||||
+1
-2
@@ -261,6 +261,7 @@ def load_scripts():
|
||||
elif issubclass(script_class, scripts_postprocessing.ScriptPostprocessing):
|
||||
postprocessing_scripts_data.append(ScriptClassData(script_class, scriptfile.path, scriptfile.basedir, module))
|
||||
|
||||
from installer import args
|
||||
for scriptfile in scripts_list:
|
||||
try:
|
||||
if scriptfile.basedir != paths.script_path:
|
||||
@@ -274,12 +275,10 @@ def load_scripts():
|
||||
current_basedir = paths.script_path
|
||||
t.record(os.path.basename(scriptfile.basedir))
|
||||
sys.path = syspath
|
||||
|
||||
global scripts_txt2img, scripts_img2img, scripts_postproc # pylint: disable=global-statement
|
||||
scripts_txt2img = ScriptRunner()
|
||||
scripts_img2img = ScriptRunner()
|
||||
scripts_postproc = scripts_postprocessing.ScriptPostprocessingRunner()
|
||||
|
||||
return t, time.time()-t0
|
||||
|
||||
|
||||
|
||||
+31
-6
@@ -20,7 +20,7 @@ import tomesd
|
||||
from transformers import logging as transformers_logging
|
||||
import ldm.modules.midas as midas
|
||||
from ldm.util import instantiate_from_config
|
||||
from modules import paths, shared, shared_items, shared_state, modelloader, devices, script_callbacks, sd_vae, sd_disable_initialization, errors, hashes, sd_models_config, sd_models_compile, sd_hijack_inpainting
|
||||
from modules import paths, shared, shared_items, shared_state, modelloader, devices, script_callbacks, sd_vae, errors, hashes, sd_models_config, sd_models_compile, sd_hijack_inpainting
|
||||
from modules.timer import Timer
|
||||
from modules.memstats import memory_stats
|
||||
from modules.paths import models_path, script_path
|
||||
@@ -603,6 +603,7 @@ def detect_pipeline(f: str, op: str = 'model', warning=True):
|
||||
warn = shared.log.warning if warning else lambda *args, **kwargs: None
|
||||
if guess == 'Autodetect':
|
||||
try:
|
||||
# guess by size
|
||||
size = round(os.path.getsize(f) / 1024 / 1024)
|
||||
if size < 128:
|
||||
warn(f'Model size smaller than expected: {f} size={size} MB')
|
||||
@@ -635,14 +636,27 @@ def detect_pipeline(f: str, op: str = 'model', warning=True):
|
||||
guess = 'Stable Diffusion XL Instruct'
|
||||
else:
|
||||
guess = 'Stable Diffusion'
|
||||
if 'LCM_' in f or 'LCM-' in f:
|
||||
# guess by name
|
||||
"""
|
||||
if 'LCM_' in f.upper() or 'LCM-' in f.upper() or '_LCM' in f.upper() or '-LCM' in f.upper():
|
||||
if shared.backend == shared.Backend.ORIGINAL:
|
||||
warn(f'Model detected as LCM model, but attempting to load using backend=original: {op}={f} size={size} MB')
|
||||
guess = 'Latent Consistency Model'
|
||||
"""
|
||||
if 'PixArt' in f:
|
||||
if shared.backend == shared.Backend.ORIGINAL:
|
||||
warn(f'Model detected as PixArt Alpha model, but attempting to load using backend=original: {op}={f} size={size} MB')
|
||||
guess = 'PixArt Alpha'
|
||||
# switch for specific variant
|
||||
if guess == 'Stable Diffusion' and 'inpaint' in f.lower():
|
||||
guess = 'Stable Diffusion Inpaint'
|
||||
elif guess == 'Stable Diffusion' and 'instruct' in f.lower():
|
||||
guess = 'Stable Diffusion Instruct'
|
||||
if guess == 'Stable Diffusion XL' and 'inpaint' in f.lower():
|
||||
guess = 'Stable Diffusion XL Inpaint'
|
||||
elif guess == 'Stable Diffusion XL' and 'instruct' in f.lower():
|
||||
guess = 'Stable Diffusion XL Instruct'
|
||||
# get actual pipeline
|
||||
pipeline = shared_items.get_pipelines().get(guess, None)
|
||||
shared.log.info(f'Autodetect: {op}="{guess}" class={pipeline.__name__} file="{f}" size={size}MB')
|
||||
except Exception as e:
|
||||
@@ -729,13 +743,13 @@ def set_diffuser_options(sd_model, vae, op: str):
|
||||
sd_model.enable_xformers_memory_efficient_attention()
|
||||
|
||||
if shared.opts.diffusers_eval:
|
||||
if hasattr(sd_model, "unet"):
|
||||
if hasattr(sd_model, "unet") and hasattr(sd_model.unet, "requires_grad_"):
|
||||
sd_model.unet.requires_grad_(False)
|
||||
sd_model.unet.eval()
|
||||
if hasattr(sd_model, "vae"):
|
||||
if hasattr(sd_model, "vae") and hasattr(sd_model.vae, "requires_grad_"):
|
||||
sd_model.vae.requires_grad_(False)
|
||||
sd_model.vae.eval()
|
||||
if hasattr(sd_model, "text_encoder"):
|
||||
if hasattr(sd_model, "text_encoder") and hasattr(sd_model.text_encoder, "requires_grad_"):
|
||||
sd_model.text_encoder.requires_grad_(False)
|
||||
sd_model.text_encoder.eval()
|
||||
|
||||
@@ -746,6 +760,10 @@ def set_diffuser_options(sd_model, vae, op: str):
|
||||
|
||||
def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument
|
||||
import torch # pylint: disable=reimported,redefined-outer-name
|
||||
if shared.cmd_opts.profile:
|
||||
import cProfile
|
||||
pr = cProfile.Profile()
|
||||
pr.enable()
|
||||
if timer is None:
|
||||
timer = Timer()
|
||||
logging.getLogger("diffusers").setLevel(logging.ERROR)
|
||||
@@ -888,6 +906,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
|
||||
shared.log.debug(f'Setting {op}: pipeline={sd_model.__class__.__name__} config={diffusers_load_config}') # pylint: disable=protected-access
|
||||
except Exception as e:
|
||||
shared.log.error(f'Diffusers failed loading: {op}={checkpoint_info.path} pipeline={shared.opts.diffusers_pipeline}/{sd_model.__class__.__name__} {e}')
|
||||
errors.display(e, f'loading {op}={checkpoint_info.path} pipeline={shared.opts.diffusers_pipeline}/{sd_model.__class__.__name__}')
|
||||
return
|
||||
else:
|
||||
shared.log.error(f'Diffusers cannot load: {op}={checkpoint_info.path}')
|
||||
@@ -961,6 +980,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
|
||||
|
||||
timer.record("load")
|
||||
devices.torch_gc(force=True)
|
||||
if shared.cmd_opts.profile:
|
||||
errors.profile(pr, 'Load')
|
||||
script_callbacks.model_loaded_callback(sd_model)
|
||||
shared.log.info(f"Load {op}: time={timer.summary()} native={get_native(sd_model)} {memory_stats()}")
|
||||
|
||||
@@ -1093,12 +1114,16 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None,
|
||||
sd_model = None
|
||||
stdout = io.StringIO()
|
||||
with contextlib.redirect_stdout(stdout):
|
||||
"""
|
||||
try:
|
||||
clip_is_included_into_sd = sd1_clip_weight in state_dict or sd2_clip_weight in state_dict
|
||||
with sd_disable_initialization.DisableInitialization(disable_clip=clip_is_included_into_sd):
|
||||
sd_model = instantiate_from_config(sd_config.model)
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
shared.log.error(f'LDM: instantiate from config: {e}')
|
||||
sd_model = instantiate_from_config(sd_config.model)
|
||||
"""
|
||||
sd_model = instantiate_from_config(sd_config.model)
|
||||
for line in stdout.getvalue().splitlines():
|
||||
if len(line) > 0:
|
||||
shared.log.info(f'LDM: {line.strip()}')
|
||||
|
||||
@@ -31,12 +31,15 @@ def setup_img2img_steps(p, steps=None):
|
||||
|
||||
|
||||
def single_sample_to_image(sample, approximation=None):
|
||||
# sample should be [4,64,64]
|
||||
if approximation is None:
|
||||
approximation = approximation_indexes.get(shared.opts.show_progress_type, None)
|
||||
if approximation is None:
|
||||
warn_once('Unknown decode type, please reset preview method')
|
||||
approximation = 0
|
||||
|
||||
if len(sample.shape) == 4 and sample.shape[0]: # likely animatediff latent
|
||||
sample = sample.permute(1, 0, 2, 3)[0]
|
||||
if approximation == 0: # Simple
|
||||
x_sample = sd_vae_approx.cheap_approximation(sample) * 0.5 + 0.5
|
||||
elif approximation == 1: # Approximate
|
||||
|
||||
+13
-8
@@ -159,7 +159,7 @@ def list_samplers():
|
||||
|
||||
def temp_disable_extensions():
|
||||
disable_safe = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-agent-scheduler', 'clip-interrogator-ext', 'stable-diffusion-webui-rembg', 'sd-extension-chainner', 'stable-diffusion-webui-images-browser']
|
||||
disable_diffusers = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris']
|
||||
disable_diffusers = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-animatediff']
|
||||
disable_original = []
|
||||
disabled = []
|
||||
if cmd_opts.safe:
|
||||
@@ -411,7 +411,7 @@ options_templates.update(options_section(('saving-images', "Image Options"), {
|
||||
|
||||
"image_sep_metadata": OptionInfo("<h2>Metadata/Logging</h2>", "", gr.HTML),
|
||||
"image_metadata": OptionInfo(True, "Include metadata in saved images"),
|
||||
"save_txt": OptionInfo(False, "Create text file next to every image with generation parameters"),
|
||||
"save_txt": OptionInfo(False, "Create info file for each every image"),
|
||||
"save_log_fn": OptionInfo("", "Create JSON log file for each saved image", component_args=hide_dirs),
|
||||
"image_watermark_enabled": OptionInfo(False, "Include watermark in saved images"),
|
||||
"image_watermark": OptionInfo('', "Image watermark string"),
|
||||
@@ -446,11 +446,12 @@ options_templates.update(options_section(('saving-paths', "Image Naming & Paths"
|
||||
|
||||
"outdir_sep_dirs": OptionInfo("<h2>Directories</h2>", "", gr.HTML),
|
||||
"outdir_samples": OptionInfo("", "Output directory for images", component_args=hide_dirs, folder=True),
|
||||
"outdir_txt2img_samples": OptionInfo("outputs/text", 'Output directory for txt2img images', component_args=hide_dirs, folder=True),
|
||||
"outdir_img2img_samples": OptionInfo("outputs/image", 'Output directory for img2img images', component_args=hide_dirs, folder=True),
|
||||
"outdir_extras_samples": OptionInfo("outputs/extras", 'Output directory for images from extras tab', component_args=hide_dirs, folder=True),
|
||||
"outdir_save": OptionInfo("outputs/save", "Directory for saving images using the Save button", component_args=hide_dirs, folder=True),
|
||||
"outdir_init_images": OptionInfo("outputs/init-images", "Directory for saving init images when using img2img", component_args=hide_dirs, folder=True),
|
||||
"outdir_txt2img_samples": OptionInfo("outputs/text", 'Directory for text generate', component_args=hide_dirs, folder=True),
|
||||
"outdir_img2img_samples": OptionInfo("outputs/image", 'Directory for image generate', component_args=hide_dirs, folder=True),
|
||||
"outdir_extras_samples": OptionInfo("outputs/extras", 'Directory for processed images', component_args=hide_dirs, folder=True),
|
||||
"outdir_save": OptionInfo("outputs/save", "Directory for manually saved images", component_args=hide_dirs, folder=True),
|
||||
"outdir_video": OptionInfo("outputs/video", "Directory for videos", component_args=hide_dirs, folder=True),
|
||||
"outdir_init_images": OptionInfo("outputs/init-images", "Directory for init images", component_args=hide_dirs, folder=True),
|
||||
|
||||
"outdir_sep_grids": OptionInfo("<h2>Grids</h2>", "", gr.HTML),
|
||||
"grid_extended_filename": OptionInfo(True, "Add extended info (seed, prompt) to filename when saving grid", gr.Checkbox, {"visible": False}),
|
||||
@@ -609,7 +610,7 @@ options_templates.update(options_section(('extra_networks', "Extra Networks"), {
|
||||
"extra_networks_styles": OptionInfo(True, "Show built-in styles"),
|
||||
"lora_preferred_name": OptionInfo("filename", "LoRA preffered name", gr.Radio, {"choices": ["filename", "alias"]}),
|
||||
"lora_add_hashes_to_infotext": OptionInfo(True, "LoRA add hash info"),
|
||||
"lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}),
|
||||
"lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 24, "step": 1}),
|
||||
"lora_functional": OptionInfo(False, "Use Kohya method for handling multiple LoRA", gr.Checkbox, { "visible": False }),
|
||||
|
||||
"sd_hypernetwork": OptionInfo("None", "Add hypernetwork to prompt", gr.Dropdown, { "choices": ["None"], "visible": False }),
|
||||
@@ -780,6 +781,7 @@ class Options:
|
||||
value = expected_type(value)
|
||||
return value
|
||||
|
||||
profiler = None
|
||||
opts = Options()
|
||||
config_filename = cmd_opts.config
|
||||
opts.load(config_filename)
|
||||
@@ -803,6 +805,7 @@ device = devices.device
|
||||
batch_cond_uncond = opts.always_batch_cond_uncond or not (cmd_opts.lowvram or cmd_opts.medvram)
|
||||
parallel_processing_allowed = not cmd_opts.lowvram
|
||||
mem_mon = modules.memmon.MemUsageMonitor("MemMon", devices.device)
|
||||
max_workers = 2
|
||||
if devices.backend == "directml":
|
||||
directml_do_hijack()
|
||||
|
||||
@@ -958,6 +961,8 @@ class Shared(sys.modules[__name__].__class__): # this class is here to provide s
|
||||
model_type = 'sd'
|
||||
elif "LatentConsistencyModel" in self.sd_model.__class__.__name__:
|
||||
model_type = 'sd' # lcm is compatible with sd
|
||||
elif "AnimateDiffPipeline" in self.sd_model.__class__.__name__:
|
||||
model_type = 'sd' # ad is compatible with sd
|
||||
elif "Kandinsky" in self.sd_model.__class__.__name__:
|
||||
model_type = 'kandinsky'
|
||||
else:
|
||||
|
||||
@@ -30,6 +30,7 @@ def get_pipelines():
|
||||
pipelines = { # note: not all pipelines can be used manually as they require prior pipeline next to decoder pipeline
|
||||
'Autodetect': None,
|
||||
'Stable Diffusion': getattr(diffusers, 'StableDiffusionPipeline', None),
|
||||
'Stable Diffusion Inpaint': getattr(diffusers, 'StableDiffusionInpaintPipeline', None),
|
||||
'Stable Diffusion Img2Img': getattr(diffusers, 'StableDiffusionImg2ImgPipeline', None),
|
||||
'Stable Diffusion Instruct': getattr(diffusers, 'StableDiffusionInstructPix2PixPipeline', None),
|
||||
'Stable Diffusion Upscale': getattr(diffusers, 'StableDiffusionUpscalePipeline', None),
|
||||
@@ -43,7 +44,7 @@ def get_pipelines():
|
||||
'Wuerstchen': getattr(diffusers, 'WuerstchenCombinedPipeline', None),
|
||||
'Kandinsky 2.1': getattr(diffusers, 'KandinskyPipeline', None),
|
||||
'Kandinsky 2.2': getattr(diffusers, 'KandinskyV22Pipeline', None),
|
||||
# 'Kandinsky 3': getattr(diffusers, 'KandinskyV3Pipeline', None),
|
||||
'Kandinsky 3': getattr(diffusers, 'Kandinsky3Pipeline', None),
|
||||
'DeepFloyd IF': getattr(diffusers, 'IFPipeline', None),
|
||||
'Custom Diffusers Pipeline': getattr(diffusers, 'DiffusionPipeline', None),
|
||||
# Segmind SSD-1B, Segmind Tiny
|
||||
|
||||
+26
-13
@@ -4,6 +4,7 @@ import re
|
||||
import os
|
||||
import csv
|
||||
import json
|
||||
import time
|
||||
from installer import log
|
||||
|
||||
|
||||
@@ -60,7 +61,7 @@ def apply_styles_to_extra(p, style: Style):
|
||||
v = type(orig)(v)
|
||||
setattr(p, k, v)
|
||||
fields.append(f'{k}={v}')
|
||||
log.info(f'Applying style: name={style.name} extra={fields}')
|
||||
log.info(f'Applying style: name="{style.name}" extra={fields}')
|
||||
|
||||
|
||||
class StyleDatabase:
|
||||
@@ -87,6 +88,7 @@ class StyleDatabase:
|
||||
|
||||
def load_style(self, fn, prefix=None):
|
||||
with open(fn, 'r', encoding='utf-8') as f:
|
||||
new_style = None
|
||||
try:
|
||||
all_styles = json.load(f)
|
||||
if type(all_styles) is dict:
|
||||
@@ -100,7 +102,7 @@ class StyleDatabase:
|
||||
name = os.path.join(prefix, name)
|
||||
else:
|
||||
name = os.path.join(os.path.dirname(os.path.relpath(fn, self.path)), name)
|
||||
self.styles[style["name"]] = Style(
|
||||
new_style = Style(
|
||||
name=name,
|
||||
desc=style.get('description', name),
|
||||
prompt=style.get("prompt", ""),
|
||||
@@ -110,26 +112,37 @@ class StyleDatabase:
|
||||
filename=fn,
|
||||
mtime=os.path.getmtime(fn),
|
||||
)
|
||||
self.styles[style["name"]] = new_style
|
||||
except Exception as e:
|
||||
log.error(f'Failed to load style: file={fn} error={e}')
|
||||
return new_style
|
||||
|
||||
|
||||
def reload(self):
|
||||
t0 = time.time()
|
||||
self.styles.clear()
|
||||
|
||||
def list_folder(folder):
|
||||
for filename in os.listdir(folder):
|
||||
fn = os.path.abspath(os.path.join(folder, filename))
|
||||
if os.path.isfile(fn) and fn.lower().endswith(".json"):
|
||||
self.load_style(fn)
|
||||
elif os.path.isdir(fn) and not fn.startswith('.'):
|
||||
list_folder(fn)
|
||||
import concurrent
|
||||
future_items = {}
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
|
||||
for filename in os.listdir(folder):
|
||||
fn = os.path.abspath(os.path.join(folder, filename))
|
||||
if os.path.isfile(fn) and fn.lower().endswith(".json"):
|
||||
future_items[executor.submit(self.load_style, fn, None)] = fn
|
||||
# self.load_style(fn)
|
||||
elif os.path.isdir(fn) and not fn.startswith('.'):
|
||||
list_folder(fn)
|
||||
self.styles = dict(sorted(self.styles.items(), key=lambda style: style[1].filename))
|
||||
if self.built_in:
|
||||
fn = os.path.join('html', 'art-styles.json')
|
||||
future_items[executor.submit(self.load_style, fn, 'built-in')] = fn
|
||||
for future in concurrent.futures.as_completed(future_items):
|
||||
future.result()
|
||||
|
||||
list_folder(self.path)
|
||||
self.styles = dict(sorted(self.styles.items(), key=lambda style: style[1].filename))
|
||||
if self.built_in:
|
||||
self.load_style(os.path.join('html', 'art-styles.json'), 'built-in')
|
||||
|
||||
log.debug(f'Load styles: folder="{self.path}" items={len(self.styles.keys())}')
|
||||
t1 = time.time()
|
||||
log.debug(f'Load styles: folder="{self.path}" items={len(self.styles.keys())} time={t1-t0:.2f}')
|
||||
|
||||
def find_style(self, name):
|
||||
found = [style for style in self.styles.values() if style.name == name]
|
||||
|
||||
+4
-1
@@ -4,7 +4,7 @@ from modules.generation_parameters_copypaste import create_override_settings_dic
|
||||
from modules.ui import plaintext_to_html
|
||||
|
||||
|
||||
def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, full_quality: bool, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_force: bool, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_steps: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, override_settings_texts, *args): # pylint: disable=unused-argument
|
||||
def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, full_quality: bool, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_force: bool, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_steps: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_center, hdr_max_boundry, override_settings_texts, *args): # pylint: disable=unused-argument
|
||||
|
||||
shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|latent_index={latent_index}|full_quality={full_quality}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}||height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_force={hr_force}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|image_cfg_scale={image_cfg_scale}|diffusers_guidance_rescale={diffusers_guidance_rescale}|refiner_steps={refiner_steps}|refiner_start={refiner_start}|refiner_prompt={refiner_prompt}|refiner_negative={refiner_negative}|override_settings_texts={override_settings_texts}')
|
||||
|
||||
@@ -57,6 +57,9 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step
|
||||
refiner_start=refiner_start,
|
||||
refiner_prompt=refiner_prompt,
|
||||
refiner_negative=refiner_negative,
|
||||
hdr_clamp=hdr_clamp, hdr_boundary=hdr_boundary, hdr_threshold=hdr_threshold,
|
||||
hdr_center=hdr_center, hdr_channel_shift=hdr_channel_shift, hdr_full_shift=hdr_full_shift,
|
||||
hdr_maximize=hdr_maximize, hdr_max_center=hdr_max_center, hdr_max_boundry=hdr_max_boundry,
|
||||
override_settings=override_settings,
|
||||
)
|
||||
p.scripts = modules.scripts.scripts_txt2img
|
||||
|
||||
+29
-14
@@ -332,7 +332,7 @@ def create_sampler_and_steps_selection(choices, tabname):
|
||||
|
||||
with FormRow(elem_classes=['flex-break']):
|
||||
sampler_index = gr.Dropdown(label='Sampling method', elem_id=f"{tabname}_sampling", choices=[x.name for x in choices], value='Default', type="index")
|
||||
steps = gr.Slider(minimum=0, maximum=99, step=1, label="Sampling steps", elem_id=f"{tabname}_steps", value=20)
|
||||
steps = gr.Slider(minimum=1, maximum=99, step=1, label="Sampling steps", elem_id=f"{tabname}_steps", value=20)
|
||||
if modules.shared.backend == modules.shared.Backend.ORIGINAL:
|
||||
with FormRow(elem_classes=['flex-break']):
|
||||
choices = ['brownian noise', 'discard penultimate sigma']
|
||||
@@ -354,7 +354,6 @@ def create_sampler_and_steps_selection(choices, tabname):
|
||||
values += ['low order'] if opts.data.get('schedulers_use_loworder', True) else []
|
||||
sampler_options = gr.CheckboxGroup(label='Sampler options', choices=choices, value=values, type='value')
|
||||
sampler_options.change(fn=set_sampler_diffuser_options, inputs=[sampler_options], outputs=[])
|
||||
|
||||
return steps, sampler_index
|
||||
|
||||
|
||||
@@ -417,16 +416,31 @@ def create_ui(startup_timer = None):
|
||||
seed, reuse_seed, subseed, reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w = create_seed_inputs('txt2img')
|
||||
|
||||
with gr.Accordion(open=False, label="Advanced", elem_id="txt2img_advanced", elem_classes=["small-accordion"]):
|
||||
with FormRow():
|
||||
cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id="txt2img_cfg_scale")
|
||||
clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=14, step=1, elem_id='txt2img_clip_skip', interactive=True)
|
||||
with FormRow():
|
||||
image_cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='Secondary CFG scale', value=6.0, elem_id="txt2img_image_cfg_scale")
|
||||
diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id="txt2img_image_cfg_rescale")
|
||||
with FormRow():
|
||||
full_quality = gr.Checkbox(label='Full quality', value=True, elem_id="txt2img_full_quality")
|
||||
restore_faces = gr.Checkbox(label='Face restore', value=False, visible=len(modules.shared.face_restorers) > 1, elem_id="txt2img_restore_faces")
|
||||
tiling = gr.Checkbox(label='Tiling', value=False, elem_id="txt2img_tiling")
|
||||
with gr.Group():
|
||||
with FormRow():
|
||||
cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id="txt2img_cfg_scale")
|
||||
clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=14, step=1, elem_id='txt2img_clip_skip', interactive=True)
|
||||
with FormRow():
|
||||
image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Secondary CFG scale', value=6.0, elem_id="txt2img_image_cfg_scale")
|
||||
diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id="txt2img_image_cfg_rescale")
|
||||
with gr.Group():
|
||||
with FormRow():
|
||||
full_quality = gr.Checkbox(label='Full quality', value=True, elem_id="txt2img_full_quality")
|
||||
restore_faces = gr.Checkbox(label='Face restore', value=False, visible=len(modules.shared.face_restorers) > 1, elem_id="txt2img_restore_faces")
|
||||
tiling = gr.Checkbox(label='Tiling', value=False, elem_id="txt2img_tiling")
|
||||
with gr.Group():
|
||||
with FormRow():
|
||||
hdr_clamp = gr.Checkbox(label='HDR clamp', value=False, elem_id="txt2img_hdr_clamp")
|
||||
hdr_boundary = gr.Slider(minimum=0.0, maximum=10.0, step=0.1, value=4.0, label='Range', elem_id="txt2img_hdr_boundary")
|
||||
hdr_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, value=0.95, label='Threshold', elem_id="txt2img_hdr_threshold")
|
||||
with FormRow():
|
||||
hdr_center = gr.Checkbox(label='HDR center', value=False, elem_id="txt2img_hdr_center")
|
||||
hdr_channel_shift = gr.Slider(minimum=0.0, maximum=2.0, step=0.1, value=1.0, label='Channel shift', elem_id="txt2img_hdr_channel_shift")
|
||||
hdr_full_shift = gr.Slider(minimum=0.0, maximum=2.0, step=0.1, value=1, label='Full shift', elem_id="txt2img_hdr_full_shift")
|
||||
with FormRow():
|
||||
hdr_maximize = gr.Checkbox(label='HDR maximize', value=False, elem_id="txt2img_hdr_maximize")
|
||||
hdr_max_center = gr.Slider(minimum=0.0, maximum=2.0, step=0.1, value=0.6, label='Center', elem_id="txt2img_hdr_max_center")
|
||||
hdr_max_boundry = gr.Slider(minimum=0.5, maximum=2.0, step=0.1, value=1.0, label='Range', elem_id="txt2img_hdr_max_boundry")
|
||||
|
||||
with gr.Accordion(open=False, label="Second pass", elem_id="txt2img_second_pass", elem_classes=["small-accordion"]):
|
||||
with FormGroup():
|
||||
@@ -493,6 +507,7 @@ def create_ui(startup_timer = None):
|
||||
enable_hr, denoising_strength,
|
||||
hr_scale, hr_upscaler, hr_force, hr_second_pass_steps, hr_resize_x, hr_resize_y,
|
||||
refiner_steps, refiner_start, refiner_prompt, refiner_negative,
|
||||
hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_center, hdr_max_boundry,
|
||||
override_settings,
|
||||
] + custom_inputs,
|
||||
outputs=[
|
||||
@@ -711,8 +726,8 @@ def create_ui(startup_timer = None):
|
||||
|
||||
with gr.Accordion(open=False, label="Advanced", elem_classes=["small-accordion"], elem_id="img2img_advanced_group"):
|
||||
with FormRow():
|
||||
cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id="img2img_cfg_scale")
|
||||
image_cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.15, label='Image CFG scale', value=1.5, elem_id="img2img_image_cfg_scale")
|
||||
cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id="img2img_cfg_scale")
|
||||
image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.15, label='Image CFG scale', value=1.5, elem_id="img2img_image_cfg_scale")
|
||||
with FormRow():
|
||||
clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=4, step=1, elem_id='img2img_clip_skip', interactive=True)
|
||||
diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id="txt2img_image_cfg_rescale")
|
||||
|
||||
+17
-6
@@ -35,16 +35,16 @@ def plaintext_to_html(text):
|
||||
|
||||
def infotext_to_html(text):
|
||||
res = parse_generation_parameters(text)
|
||||
prompt = res.get('Prompt', None)
|
||||
negative = res.get('Negative prompt', None)
|
||||
prompt = res.get('Prompt', '')
|
||||
negative = res.get('Negative prompt', '')
|
||||
res.pop('Prompt', None)
|
||||
res.pop('Negative prompt', None)
|
||||
params = [f'{k}: {v}' for k, v in res.items() if v is not None]
|
||||
params = '| '.join(params)
|
||||
params = '| '.join(params) if len(params) > 0 else ''
|
||||
code = f'''
|
||||
<p><b>Prompt:</b> {prompt}</p>
|
||||
<p><b>Negative:</b> {negative}</p>
|
||||
<p><b>Parameters:</b> {params}</p>
|
||||
<p><b>Prompt:</b> {html.escape(prompt)}</p>
|
||||
<p><b>Negative:</b> {html.escape(negative)}</p>
|
||||
<p><b>Parameters:</b> {html.escape(params)}</p>
|
||||
'''
|
||||
return code
|
||||
|
||||
@@ -134,6 +134,17 @@ def save_files(js_data, images, html_info, index):
|
||||
shutil.copy(fullfn, destination)
|
||||
shared.log.info(f'Copying image: file="{fullfn}" folder="{destination}"')
|
||||
tgt_filename = os.path.join(destination, os.path.basename(fullfn))
|
||||
if shared.opts.save_txt:
|
||||
try:
|
||||
from PIL import Image
|
||||
image = Image.open(fullfn)
|
||||
info, _ = images.read_info_from_image(image)
|
||||
filename_txt = f"{os.path.splitext(tgt_filename)[0]}.txt"
|
||||
with open(filename_txt, "w", encoding="utf8") as file:
|
||||
file.write(f"{info}\n")
|
||||
shared.log.debug(f'Saving: text="{filename_txt}"')
|
||||
except Exception as e:
|
||||
shared.log.warning(f'Image description save failed: {filename_txt} {e}')
|
||||
modules.script_callbacks.image_save_btn_callback(tgt_filename)
|
||||
else:
|
||||
image = image_from_url_text(filedata)
|
||||
|
||||
@@ -15,7 +15,7 @@ from collections import OrderedDict
|
||||
import gradio as gr
|
||||
from PIL import Image
|
||||
from starlette.responses import FileResponse, JSONResponse
|
||||
from modules import paths, shared, scripts, modelloader
|
||||
from modules import paths, shared, scripts, modelloader, errors
|
||||
from modules.ui_components import ToolButton
|
||||
import modules.ui_symbols as symbols
|
||||
|
||||
@@ -270,7 +270,7 @@ class ExtraNetworksPage:
|
||||
self.html = f"<div id='{tabname}_{self_name_id}_subdirs' class='extra-network-subdirs'>{subdirs_html}</div><div id='{tabname}_{self_name_id}_cards' class='extra-network-cards'>{self.html}</div>"
|
||||
else:
|
||||
return ''
|
||||
shared.log.debug(f"Extra networks: page='{self.name}' items={len(self.items)} subfolders={len(subdirs)} tab={tabname} folders={self.allowed_directories_for_previews()} list={self.list_time:.2f} desc={self.desc_time:.2f} info={self.info_time:.2f}")
|
||||
shared.log.debug(f"Extra networks: page='{self.name}' items={len(self.items)} subfolders={len(subdirs)} tab={tabname} folders={self.allowed_directories_for_previews()} list={self.list_time:.2f} desc={self.desc_time:.2f} info={self.info_time:.2f} workers={shared.max_workers}")
|
||||
if len(self.missing_thumbs) > 0:
|
||||
threading.Thread(target=self.create_thumb).start()
|
||||
return self.html
|
||||
@@ -463,6 +463,10 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
|
||||
ui.tabs = gr.Tabs(elem_id=tabname+"_extra_tabs")
|
||||
ui.button_details = gr.Button('Details', elem_id=tabname+"_extra_details_btn", visible=False)
|
||||
state = {}
|
||||
if shared.cmd_opts.profile:
|
||||
import cProfile
|
||||
pr = cProfile.Profile()
|
||||
pr.enable()
|
||||
|
||||
def get_item(state, params = None):
|
||||
if params is not None and type(params) == dict:
|
||||
@@ -567,6 +571,10 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
|
||||
page_html = gr.HTML(page.html, elem_id=f'{tabname}{page.name}_extra_page', elem_classes="extra-networks-page")
|
||||
ui.pages.append(page_html)
|
||||
tab.select(ui_tab_change, _js="getENActivePage", inputs=[ui.button_details], outputs=[ui.button_scan, ui.button_save, ui.button_model])
|
||||
if shared.cmd_opts.profile:
|
||||
errors.profile(pr, 'ExtraNetworks')
|
||||
pr.disable()
|
||||
|
||||
# ui.tabs.change(fn=ui_tab_change, inputs=[], outputs=[ui.button_scan, ui.button_save])
|
||||
|
||||
def fn_save_img(image):
|
||||
|
||||
@@ -64,7 +64,7 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage):
|
||||
return record
|
||||
|
||||
def list_items(self):
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor:
|
||||
future_items = {executor.submit(self.create_item, cp): cp for cp in list(sd_models.checkpoints_list.copy())}
|
||||
for future in concurrent.futures.as_completed(future_items):
|
||||
item = future.result()
|
||||
|
||||
@@ -95,7 +95,7 @@ class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage):
|
||||
return item
|
||||
|
||||
def list_items(self):
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor:
|
||||
future_items = {executor.submit(self.create_item, style): style for style in list(shared.prompt_styles.styles)}
|
||||
for future in concurrent.futures.as_completed(future_items):
|
||||
item = future.result()
|
||||
|
||||
@@ -68,7 +68,7 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage):
|
||||
self.embeddings = []
|
||||
self.embeddings = sorted(self.embeddings, key=lambda emb: emb.filename)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor:
|
||||
future_items = {executor.submit(self.create_item, net): net for net in self.embeddings}
|
||||
for future in concurrent.futures.as_completed(future_items):
|
||||
item = future.result()
|
||||
|
||||
@@ -81,7 +81,7 @@ def create_ui():
|
||||
custom_name = gr.Textbox(label="New model name")
|
||||
with FormRow():
|
||||
merge_mode = gr.Dropdown(choices=merge_methods.__all__, value="weighted_sum", label="Interpolation Method")
|
||||
merge_mode_docs = gr.HTML(value=getattr(merge_methods, "weighted_sum").__doc__.replace("\n", "<br>"))
|
||||
merge_mode_docs = gr.HTML(value=getattr(merge_methods, "weighted_sum", "").__doc__.replace("\n", "<br>"))
|
||||
with FormRow():
|
||||
primary_model_name = gr.Dropdown(sd_model_choices(), label="Primary model", value="None")
|
||||
create_refresh_button(primary_model_name, sd_models.list_models, lambda: {"choices": sd_model_choices()}, "refresh_checkpoint_A")
|
||||
@@ -641,7 +641,7 @@ def create_ui():
|
||||
civit_update_download_btn = gr.Button(value="Download", variant='primary', visible=False)
|
||||
|
||||
class CivitModel:
|
||||
def __init__(self, name, fn, sha = None, meta = {}):
|
||||
def __init__(self, name, fn, sha = None, meta = {}): # noqa: B006
|
||||
self.name = name
|
||||
self.id = meta.get('id', 0)
|
||||
self.fn = fn
|
||||
@@ -701,7 +701,7 @@ def create_ui():
|
||||
model.latest_name = f.get('name', '')
|
||||
if model.vername == model.latest:
|
||||
model.status = 'Latest'
|
||||
elif any(map(lambda v: v in model.latest_hashes, all_hashes)):
|
||||
elif any(map(lambda v: v in model.latest_hashes, all_hashes)): # pylint: disable=cell-var-from-loop # noqa: C417
|
||||
model.status = 'Downloaded'
|
||||
else:
|
||||
model.status = 'Available'
|
||||
@@ -716,7 +716,7 @@ def create_ui():
|
||||
nonlocal selected_model, update_data
|
||||
try:
|
||||
selected_model = [m for m in update_data if m.fn == in_data[evt.index[0]][1]][0]
|
||||
except:
|
||||
except Exception:
|
||||
selected_model = None
|
||||
if selected_model is None or selected_model.url is None or selected_model.status != 'Available':
|
||||
return [gr.update(value='Model update not available'), gr.update(visible=False)]
|
||||
|
||||
@@ -81,7 +81,7 @@ class DDPM(pl.LightningModule):
|
||||
super().__init__()
|
||||
assert parameterization in ["eps", "x0", "v"], 'currently only supporting "eps" and "x0" and "v"'
|
||||
self.parameterization = parameterization
|
||||
print(f"{self.__class__.__name__}: Running in {self.parameterization}-prediction mode")
|
||||
print(f"{self.__class__.__name__}: mode={self.parameterization}")
|
||||
self.cond_stage_model = None
|
||||
self.clip_denoised = clip_denoised
|
||||
self.log_every_t = log_every_t
|
||||
|
||||
@@ -301,8 +301,8 @@ class FrozenCLIPT5Encoder(AbstractEncoder):
|
||||
super().__init__()
|
||||
self.clip_encoder = FrozenCLIPEmbedder(clip_version, device, max_length=clip_max_length)
|
||||
self.t5_encoder = FrozenT5Embedder(t5_version, device, max_length=t5_max_length)
|
||||
print(f"{self.clip_encoder.__class__.__name__} has {count_params(self.clip_encoder) * 1.e-6:.2f} M parameters, "
|
||||
f"{self.t5_encoder.__class__.__name__} comes with {count_params(self.t5_encoder) * 1.e-6:.2f} M params.")
|
||||
print(f"{self.clip_encoder.__class__.__name__} params={count_params(self.clip_encoder) * 1.e-6:.2f} M "
|
||||
f"{self.t5_encoder.__class__.__name__} params={count_params(self.t5_encoder) * 1.e-6:.2f} M")
|
||||
|
||||
def encode(self, text):
|
||||
return self(text)
|
||||
|
||||
@@ -75,7 +75,7 @@ def mean_flat(tensor):
|
||||
def count_params(model, verbose=False):
|
||||
total_params = sum(p.numel() for p in model.parameters())
|
||||
if verbose:
|
||||
print(f"{model.__class__.__name__} has {total_params*1.e-6:.2f} M params.")
|
||||
print(f"{model.__class__.__name__} params={total_params*1.e-6:.2f}M")
|
||||
return total_params
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -51,7 +51,7 @@ requests==2.31.0
|
||||
tqdm==4.66.1
|
||||
accelerate==0.24.1
|
||||
opencv-python-headless==4.7.0.72
|
||||
diffusers==0.23.1
|
||||
diffusers==0.24.0
|
||||
einops==0.4.1
|
||||
gradio==3.43.2
|
||||
huggingface_hub==0.19.4
|
||||
@@ -61,6 +61,7 @@ numba==0.57.1
|
||||
pandas==1.5.3
|
||||
protobuf==3.20.3
|
||||
pytorch_lightning==1.9.4
|
||||
tokenizers==0.15.0
|
||||
transformers==4.35.2
|
||||
tomesd==0.1.3
|
||||
urllib3==1.26.15
|
||||
@@ -68,3 +69,4 @@ Pillow==10.1.0
|
||||
timm==0.9.7
|
||||
pydantic==1.10.13
|
||||
typing-extensions==4.8.0
|
||||
peft
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
Lightweight AnimateDiff implementation in Diffusers
|
||||
Docs: <https://huggingface.co/docs/diffusers/api/pipelines/animatediff>
|
||||
TODO:
|
||||
- SDXL
|
||||
- Custom models
|
||||
- Custom LORAs
|
||||
- Enable second pass
|
||||
- TemporalDiff: https://huggingface.co/CiaraRowles/TemporalDiff/tree/main
|
||||
- AnimateFace: https://huggingface.co/nlper2022/animatediff_face_512/tree/main
|
||||
"""
|
||||
|
||||
import gradio as gr
|
||||
import diffusers
|
||||
from modules import scripts, processing, shared, devices, sd_models
|
||||
|
||||
|
||||
# config
|
||||
ADAPTERS = {
|
||||
'None': None,
|
||||
'Motion 1.4': 'guoyww/animatediff-motion-adapter-v1-4',
|
||||
'Motion 1.5 v1': 'guoyww/animatediff-motion-adapter-v1-5',
|
||||
'Motion 1.5 v2' :'guoyww/animatediff-motion-adapter-v1-5-2',
|
||||
# 'Motion SD-XL Beta v1' :'vladmandic/animatediff-sdxl',
|
||||
'TemporalDiff': 'vladmandic/temporaldiff',
|
||||
'AnimateFace': 'vladmandic/animateface',
|
||||
}
|
||||
LORAS = {
|
||||
'None': None,
|
||||
'Zoom-in': 'guoyww/animatediff-motion-lora-zoom-in',
|
||||
'Zoom-out': 'guoyww/animatediff-motion-lora-zoom-out',
|
||||
'Pan-left': 'guoyww/animatediff-motion-lora-pan-left',
|
||||
'Pan-right': 'guoyww/animatediff-motion-lora-pan-right',
|
||||
'Tilt-up': 'guoyww/animatediff-motion-lora-tilt-up',
|
||||
'Tilt-down': 'guoyww/animatediff-motion-lora-tilt-down',
|
||||
'Roll-left': 'guoyww/animatediff-motion-lora-rolling-anticlockwise',
|
||||
'Roll-right': 'guoyww/animatediff-motion-lora-rolling-clockwise',
|
||||
}
|
||||
|
||||
# state
|
||||
motion_adapter = None # instance of diffusers.MotionAdapter
|
||||
loaded_adapter = None # name of loaded adapter
|
||||
orig_pipe = None # original sd_model pipeline
|
||||
|
||||
|
||||
def set_adapter(adapter_name: str = 'None'):
|
||||
if shared.sd_model is None:
|
||||
return
|
||||
if shared.backend != shared.Backend.DIFFUSERS:
|
||||
shared.log.warning('AnimateDiff: not in diffusers mode')
|
||||
return
|
||||
global motion_adapter, loaded_adapter, orig_pipe # pylint: disable=global-statement
|
||||
# adapter_name = name if name is not None and isinstance(name, str) else loaded_adapter
|
||||
if adapter_name is None or adapter_name == 'None' or shared.sd_model is None:
|
||||
motion_adapter = None
|
||||
loaded_adapter = None
|
||||
if orig_pipe is not None:
|
||||
shared.log.debug(f'AnimateDiff restore pipeline: adapter="{loaded_adapter}"')
|
||||
shared.sd_model = orig_pipe
|
||||
orig_pipe = None
|
||||
return
|
||||
if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
|
||||
shared.log.warning(f'AnimateDiff: unsupported model type: {shared.sd_model.__class__.__name__}')
|
||||
return
|
||||
if motion_adapter is not None and loaded_adapter == adapter_name and shared.sd_model.__class__.__name__ == 'AnimateDiffPipeline':
|
||||
shared.log.debug(f'AnimateDiff cache: adapter="{adapter_name}"')
|
||||
return
|
||||
if getattr(shared.sd_model, 'image_encoder', None) is not None:
|
||||
shared.log.debug('AnimateDiff: unloading IP adapter')
|
||||
# shared.sd_model.image_encoder = None
|
||||
shared.sd_model.unet.set_default_attn_processor()
|
||||
shared.sd_model.unet.config.encoder_hid_dim_type = None
|
||||
try:
|
||||
shared.log.info(f'AnimateDiff load: adapter="{adapter_name}"')
|
||||
motion_adapter = None
|
||||
motion_adapter = diffusers.MotionAdapter.from_pretrained(adapter_name, cache_dir=shared.opts.diffusers_dir, torch_dtype=devices.dtype, low_cpu_mem_usage=False, device_map=None)
|
||||
motion_adapter.to(shared.device)
|
||||
sd_models.set_diffuser_options(motion_adapter, vae=None, op='adapter')
|
||||
loaded_adapter = adapter_name
|
||||
|
||||
new_pipe = diffusers.AnimateDiffPipeline(
|
||||
vae=shared.sd_model.vae,
|
||||
text_encoder=shared.sd_model.text_encoder,
|
||||
tokenizer=shared.sd_model.tokenizer,
|
||||
unet=shared.sd_model.unet,
|
||||
scheduler=shared.sd_model.scheduler,
|
||||
motion_adapter=motion_adapter,
|
||||
)
|
||||
orig_pipe = shared.sd_model
|
||||
new_pipe.sd_checkpoint_info = shared.sd_model.sd_checkpoint_info
|
||||
new_pipe.sd_model_hash = shared.sd_model.sd_model_hash
|
||||
new_pipe.sd_model_checkpoint = shared.sd_model.sd_checkpoint_info.filename
|
||||
new_pipe.is_sdxl = False
|
||||
new_pipe.is_sd2 = False
|
||||
new_pipe.is_sd1 = True
|
||||
shared.sd_model = new_pipe
|
||||
shared.sd_model.to(shared.device)
|
||||
sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model')
|
||||
shared.log.debug(f'AnimateDiff create pipeline: adapter="{loaded_adapter}"')
|
||||
except Exception as e:
|
||||
motion_adapter = None
|
||||
loaded_adapter = None
|
||||
shared.log.error(f'AnimateDiff load error: adapter="{adapter_name}" {e}')
|
||||
|
||||
|
||||
class Script(scripts.Script):
|
||||
def title(self):
|
||||
return 'AnimateDiff'
|
||||
|
||||
def show(self, _is_img2img):
|
||||
return scripts.AlwaysVisible if shared.backend == shared.Backend.DIFFUSERS else False
|
||||
|
||||
|
||||
def ui(self, _is_img2img):
|
||||
def video_type_change(video_type):
|
||||
return [
|
||||
gr.update(visible=video_type != 'None'),
|
||||
gr.update(visible=video_type == 'GIF' or video_type == 'PNG'),
|
||||
gr.update(visible=video_type == 'MP4'),
|
||||
gr.update(visible=video_type == 'MP4'),
|
||||
]
|
||||
|
||||
with gr.Accordion('AnimateDiff', open=False, elem_id='animatediff'):
|
||||
with gr.Row():
|
||||
adapter_index = gr.Dropdown(label='Adapter', choices=list(ADAPTERS), value='None')
|
||||
frames = gr.Slider(label='Frames', minimum=1, maximum=32, step=1, value=16)
|
||||
with gr.Row():
|
||||
lora_index = gr.Dropdown(label='Lora', choices=list(LORAS), value='None')
|
||||
strength = gr.Slider(label='Strength', minimum=0.0, maximum=2.0, step=0.05, value=1.0)
|
||||
with gr.Row():
|
||||
latent_mode = gr.Checkbox(label='Latent mode', value=False)
|
||||
with gr.Row():
|
||||
video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None')
|
||||
duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False)
|
||||
with gr.Row():
|
||||
gif_loop = gr.Checkbox(label='Loop', value=True, visible=False)
|
||||
mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False)
|
||||
mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False)
|
||||
video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate])
|
||||
return [adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate]
|
||||
|
||||
def process(self, p: processing.StableDiffusionProcessing, adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
|
||||
adapter = ADAPTERS[adapter_index]
|
||||
lora = LORAS[lora_index]
|
||||
set_adapter(adapter)
|
||||
if motion_adapter is None:
|
||||
return
|
||||
shared.log.debug(f'AnimateDiff: adapter="{adapter}" lora="{lora}" strength={strength} video={video_type}')
|
||||
if lora is not None and lora != 'None':
|
||||
shared.sd_model.load_lora_weights(lora, adapter_name=lora)
|
||||
shared.sd_model.set_adapters([lora], adapter_weights=[strength])
|
||||
p.extra_generation_params['AnimateDiff Lora'] = f'{lora}:{strength}'
|
||||
p.extra_generation_params['AnimateDiff'] = loaded_adapter
|
||||
p.do_not_save_grid = True
|
||||
p.task_args['num_frames'] = frames
|
||||
p.task_args['num_inference_steps'] = p.steps
|
||||
if not latent_mode:
|
||||
p.task_args['output_type'] = 'np'
|
||||
|
||||
def postprocess(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
|
||||
from modules.images import save_video
|
||||
if video_type != 'None':
|
||||
save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
|
||||
@@ -0,0 +1,104 @@
|
||||
"""
|
||||
Lightweight IP-Adapter applied to existing pipeline in Diffusers
|
||||
- Downloads image_encoder or first usage (2.5GB)
|
||||
- Introduced via: https://github.com/huggingface/diffusers/pull/5713
|
||||
- IP adapters: https://huggingface.co/h94/IP-Adapter
|
||||
TODO:
|
||||
- Additional IP addapters
|
||||
- SD/SDXL autodetect
|
||||
"""
|
||||
|
||||
import gradio as gr
|
||||
from modules import scripts, processing, shared, devices
|
||||
|
||||
|
||||
image_encoder = None
|
||||
loaded = None
|
||||
ADAPTERS = [
|
||||
'none',
|
||||
'models/ip-adapter_sd15',
|
||||
'models/ip-adapter_sd15_light',
|
||||
# 'models/ip-adapter_sd15_vit-G', # RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x1024 and 1280x3072)
|
||||
# 'models/ip-adapter-plus_sd15', # KeyError: 'proj.weight'
|
||||
# 'models/ip-adapter-plus-face_sd15', # KeyError: 'proj.weight'
|
||||
# 'models/ip-adapter-full-face_sd15', # KeyError: 'proj.weight'
|
||||
'sdxl_models/ip-adapter_sdxl',
|
||||
# 'sdxl_models/ip-adapter_sdxl_vit-h',
|
||||
# 'sdxl_models/ip-adapter-plus_sdxl_vit-h',
|
||||
# 'sdxl_models/ip-adapter-plus-face_sdxl_vit-h',
|
||||
]
|
||||
|
||||
|
||||
class Script(scripts.Script):
|
||||
def title(self):
|
||||
return 'IP Adapter'
|
||||
|
||||
def show(self, is_img2img):
|
||||
return scripts.AlwaysVisible if shared.backend == shared.Backend.DIFFUSERS else False
|
||||
|
||||
def ui(self, _is_img2img):
|
||||
with gr.Accordion('IP Adapter', open=False, elem_id='ipadapter'):
|
||||
with gr.Row():
|
||||
adapter = gr.Dropdown(label='Adapter', choices=ADAPTERS, value='none')
|
||||
scale = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=0.5)
|
||||
with gr.Row():
|
||||
image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512)
|
||||
return [adapter, scale, image]
|
||||
|
||||
def process(self, p: processing.StableDiffusionProcessing, adapter, scale, image): # pylint: disable=arguments-differ
|
||||
import torch
|
||||
from transformers import CLIPVisionModelWithProjection
|
||||
|
||||
# init code
|
||||
global loaded, image_encoder # pylint: disable=global-statement
|
||||
if shared.sd_model is None:
|
||||
return
|
||||
if shared.backend != shared.Backend.DIFFUSERS:
|
||||
shared.log.warning('IP adapter: not in diffusers mode')
|
||||
return
|
||||
if adapter == 'none':
|
||||
if hasattr(shared.sd_model, 'set_ip_adapter_scale'):
|
||||
shared.sd_model.set_ip_adapter_scale(0)
|
||||
if loaded is not None:
|
||||
shared.log.debug('IP adapter: unload attention processor')
|
||||
shared.sd_model.unet.set_default_attn_processor()
|
||||
shared.sd_model.unet.config.encoder_hid_dim_type = None
|
||||
loaded = None
|
||||
return
|
||||
if image is None:
|
||||
shared.log.error('IP adapter: no image')
|
||||
return
|
||||
if not hasattr(shared.sd_model, 'load_ip_adapter'):
|
||||
shared.log.error(f'IP adapter: pipeline not supported: {shared.sd_model.__class__.__name__}')
|
||||
return
|
||||
if getattr(shared.sd_model, 'image_encoder', None) is None:
|
||||
if shared.sd_model_type == 'sd':
|
||||
subfolder = 'models/image_encoder'
|
||||
elif shared.sd_model_type == 'sdxl':
|
||||
subfolder = 'sdxl_models/image_encoder'
|
||||
else:
|
||||
shared.log.error(f'IP adapter: unsupported model type: {shared.sd_model_type}')
|
||||
return
|
||||
if image_encoder is None:
|
||||
try:
|
||||
image_encoder = CLIPVisionModelWithProjection.from_pretrained("h94/IP-Adapter", subfolder=subfolder, torch_dtype=torch.float16, cache_dir=shared.opts.diffusers_dir, use_safetensors=True).to(devices.device)
|
||||
except Exception as e:
|
||||
shared.log.error(f'IP adapter: failed to load image encoder: {e}')
|
||||
return
|
||||
|
||||
# main code
|
||||
subfolder, model = adapter.split('/')
|
||||
if model != loaded or getattr(shared.sd_model.unet.config, 'encoder_hid_dim_type', None) is None:
|
||||
if loaded is not None:
|
||||
shared.log.debug('IP adapter: reset attention processor')
|
||||
shared.sd_model.unet.set_default_attn_processor()
|
||||
loaded = None
|
||||
shared.log.info(f'IP adapter load: adapter="{model}" scale={scale} image={image}')
|
||||
shared.sd_model.image_encoder = image_encoder
|
||||
shared.sd_model.load_ip_adapter("h94/IP-Adapter", subfolder=subfolder, weight_name=f'{model}.safetensors')
|
||||
loaded = model
|
||||
else:
|
||||
shared.log.debug(f'IP adapter cache: adapter="{model}" scale={scale} image={image}')
|
||||
shared.sd_model.set_ip_adapter_scale(scale)
|
||||
p.task_args['ip_adapter_image'] = p.batch_size * [image]
|
||||
p.extra_generation_params["IP Adapter"] = f'{adapter}:{scale}'
|
||||
@@ -0,0 +1,78 @@
|
||||
"""
|
||||
Additional params for StableVideoDiffusion
|
||||
"""
|
||||
import torch
|
||||
import gradio as gr
|
||||
from modules import scripts, processing, shared, sd_models, images
|
||||
|
||||
|
||||
class Script(scripts.Script):
|
||||
def title(self):
|
||||
return 'Stable Video Diffusion'
|
||||
|
||||
def show(self, is_img2img):
|
||||
return is_img2img if shared.backend == shared.Backend.DIFFUSERS else False
|
||||
|
||||
# return signature is array of gradio components
|
||||
def ui(self, _is_img2img):
|
||||
def video_type_change(video_type):
|
||||
return [
|
||||
gr.update(visible=video_type != 'None'),
|
||||
gr.update(visible=video_type == 'GIF' or video_type == 'PNG'),
|
||||
gr.update(visible=video_type == 'MP4'),
|
||||
gr.update(visible=video_type == 'MP4'),
|
||||
]
|
||||
|
||||
with gr.Row():
|
||||
num_frames = gr.Slider(label='Frames', minimum=1, maximum=50, step=1, value=14)
|
||||
min_guidance_scale = gr.Slider(label='Min guidance', minimum=0.0, maximum=10.0, step=0.1, value=1.0)
|
||||
max_guidance_scale = gr.Slider(label='Max guidance', minimum=0.0, maximum=10.0, step=0.1, value=3.0)
|
||||
with gr.Row():
|
||||
decode_chunk_size = gr.Slider(label='Decode chunks', minimum=1, maximum=25, step=1, value=6)
|
||||
motion_bucket_id = gr.Slider(label='Motion level', minimum=0, maximum=1, step=0.05, value=0.5)
|
||||
noise_aug_strength = gr.Slider(label='Noise strength', minimum=0.0, maximum=1.0, step=0.01, value=0.1)
|
||||
with gr.Row():
|
||||
override_resolution = gr.Checkbox(label='Override resolution', value=True)
|
||||
with gr.Row():
|
||||
video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None')
|
||||
duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False)
|
||||
with gr.Row():
|
||||
gif_loop = gr.Checkbox(label='Loop', value=True, visible=False)
|
||||
mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False)
|
||||
mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False)
|
||||
video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate])
|
||||
return [num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate]
|
||||
|
||||
def run(self, p: processing.StableDiffusionProcessing, num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
|
||||
if shared.sd_model is None or shared.sd_model.__class__.__name__ != 'StableVideoDiffusionPipeline':
|
||||
return None
|
||||
if hasattr(p, 'init_images') and len(p.init_images) > 0:
|
||||
if override_resolution:
|
||||
p.width = 1024
|
||||
p.height = 576
|
||||
p.task_args['image'] = images.resize_image(resize_mode=2, im=p.init_images[0], width=p.width, height=p.height, upscaler_name=None, output_type='pil')
|
||||
else:
|
||||
p.task_args['image'] = p.init_images[0]
|
||||
p.ops.append('svd')
|
||||
p.do_not_save_grid = True
|
||||
p.sampler_name = 'Default' # svd does not support non-default sampler
|
||||
p.task_args['generator'] = torch.manual_seed(p.seed) # svd does not support gpu based generator
|
||||
p.task_args['width'] = p.width
|
||||
p.task_args['height'] = p.height
|
||||
p.task_args['num_frames'] = num_frames
|
||||
p.task_args['decode_chunk_size'] = decode_chunk_size
|
||||
p.task_args['motion_bucket_id'] = round(255 * motion_bucket_id)
|
||||
p.task_args['noise_aug_strength'] = noise_aug_strength
|
||||
p.task_args['num_inference_steps'] = p.steps
|
||||
p.task_args['min_guidance_scale'] = min_guidance_scale
|
||||
p.task_args['max_guidance_scale'] = max_guidance_scale
|
||||
p.task_args['output_type'] = 'np'
|
||||
shared.log.debug(f'StableVideo: args={p.task_args}')
|
||||
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
|
||||
processed = processing.process_images(p)
|
||||
if video_type != 'None':
|
||||
images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
|
||||
return processed
|
||||
else:
|
||||
shared.log.error('StableVideo: no init_images')
|
||||
return None
|
||||
@@ -257,6 +257,12 @@ axis_options = [
|
||||
AxisOption("[Refiner] Model", str, apply_refiner, fmt=format_value, cost=1.0, choices=lambda: ['None'] + sorted(sd_models.checkpoints_list)),
|
||||
AxisOption("[Refiner] Refiner start", float, apply_field("refiner_start")),
|
||||
AxisOption("[Refiner] Refiner steps", float, apply_field("refiner_steps")),
|
||||
AxisOption("[HDR] Clamp boundary", float, apply_field("hdr_boundary")),
|
||||
AxisOption("[HDR] Clamp threshold", float, apply_field("hdr_threshold")),
|
||||
AxisOption("[HDR] Center channel shift", float, apply_field("hdr_channel_shift")),
|
||||
AxisOption("[HDR] Center full shift", float, apply_field("hdr_full_shift")),
|
||||
AxisOption("[HDR] Maximize center shift", float, apply_field("hdr_max_center")),
|
||||
AxisOption("[HDR] Maximize boundary", float, apply_field("hdr_max_boundry")),
|
||||
AxisOption("[ToMe] Token merging ratio (txt2img)", float, apply_override('token_merging_ratio')),
|
||||
AxisOption("[ToMe] Token merging ratio (hires)", float, apply_override('token_merging_ratio_hr')),
|
||||
AxisOption("[FreeU] 1st stage backbone factor", float, apply_setting('freeu_b1')),
|
||||
|
||||
@@ -47,8 +47,6 @@ state = shared.state
|
||||
backend = shared.backend
|
||||
if not modules.loader.initialized:
|
||||
timer.startup.record("libraries")
|
||||
log.setLevel(logging.DEBUG if cmd_opts.debug else logging.INFO)
|
||||
logging.disable(logging.NOTSET if cmd_opts.debug else logging.DEBUG)
|
||||
if cmd_opts.server_name:
|
||||
server_name = cmd_opts.server_name
|
||||
else:
|
||||
@@ -212,7 +210,6 @@ def async_policy():
|
||||
|
||||
def start_common():
|
||||
log.debug('Entering start sequence')
|
||||
logging.disable(logging.NOTSET if cmd_opts.debug else logging.DEBUG)
|
||||
if shared.cmd_opts.data_dir is not None and len(shared.cmd_opts.data_dir) > 0:
|
||||
log.info(f'Using data path: {shared.cmd_opts.data_dir}')
|
||||
if shared.cmd_opts.models_dir is not None and len(shared.cmd_opts.models_dir) > 0 and shared.cmd_opts.models_dir != 'models':
|
||||
@@ -314,6 +311,9 @@ def webui(restart=False):
|
||||
modules.sd_models.write_metadata()
|
||||
load_model()
|
||||
shared.opts.save(shared.config_filename)
|
||||
if cmd_opts.profile:
|
||||
for k, v in modules.script_callbacks.callback_map.items():
|
||||
shared.log.debug(f'Registered callbacks: {k}={len(v)} {[c.script for c in v]}')
|
||||
log.info(f"Startup time: {timer.startup.summary()}")
|
||||
debug = log.info if os.environ.get('SD_SCRIPT_DEBUG', None) is not None else lambda *args, **kwargs: None
|
||||
debug('Loaded scripts:')
|
||||
|
||||
+1
-1
Submodule wiki updated: 6f0a39edad...931082304d
Reference in New Issue
Block a user