diff --git a/CHANGELOG.md b/CHANGELOG.md index 21c1a45ea..6650290b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,67 @@ # Change Log for SD.Next +## Update for 2023-12-04 + +What's new? Native video in SD.Next via both **AnimateDiff** and **Stable-Video-Diffusion** - and including native MP4 encoding and smooth video outputs out-of-the-box, not just animated-GIFs. +Also new is support for **SDXL-Turbo** as well as new **Kandinsky 3** models and cool latent correction via **HDR controls** for any *txt2img* workflows, best-of-class **SDXL model merge** using full ReBasin methods and further mobile UI optimizations. + +- **Diffusers** + - **IP adapter** + - lightweight native implementation of T2I adapters which can guide generation towards specific image style + - supports most T2I models, not limited to SD 1.5 + - models are auto-downloaded on first use + - for IP adapter support in *Original* backend, use standard *ControlNet* extension + - **AnimateDiff** + - lightweight native implementation of AnimateDiff models: + *AnimateDiff 1.4, 1.5 v1, 1.5 v2, AnimateFace* + - supports SD 1.5 only + - models are auto-downloaded on first use + - for video saving support, see video support section + - can be combined with IP-Adapter for even better results! + - for AnimateDiff support in *Original* backend, use standard *AnimateDiff* extension + - **HDR latent control**, based on [article](https://huggingface.co/blog/TimothyAlexisVass/explaining-the-sdxl-latent-space#long-prompts-at-high-guidance-scales-becoming-possible) + - in *Advanced* params + - allows control of *latent clamping*, *color centering* and *range maximimization* + - supported by *XYZ grid* + - [SD21 Turbo](https://huggingface.co/stabilityai/sd-turbo) and [SDXL Turbo]() support + - just set CFG scale (0.0-1.0) and steps (1-3) to a very low value + - compatible with original StabilityAI SDXL-Turbo or any of the newer merges + - download safetensors or select from networks -> reference + - [Stable Video Diffusion](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid) and [Stable Video Diffusion XT](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid-xt) support + - download using built-in model downloader or simply select from *networks -> reference* + support for manually downloaded safetensors models will be added later + - for video saving support, see video support section + - go to *image* tab, enter input image and select *script* -> *stable video diffusion* + - [Kandinsky 3](https://huggingface.co/kandinsky-community/kandinsky-3) support + - download using built-in model downloader or simply select from *networks -> reference* + - this model is absolutely massive at 27.5GB at fp16, so be patient + - model params count is at 11.9B (compared to SD-XL at 3.3B) and its trained on mixed resolutions from 256px to 1024px + - use either model offload or sequential cpu offload to be able to use it + - better autodetection of *inpaint* and *instruct* pipelines + - support long seconary prompt for refiner +- **Video support** + - applies to any model that supports video generation, e.g. AnimateDiff and StableVideoDiffusion + - support for **animated-GIF**, **animated-PNG** and **MP4** + - GIF and PNG can be looped + - MP4 can have additional padding at the start/end as well as motion-aware interpolated frames for smooth playback + interpolation is done using [RIFE](https://arxiv.org/abs/2011.06294) with native implementation in SD.Next + And its fast - interpolation from 16 frames with 10x frames to target 160 frames results takes 2-3sec + - output folder for videos is in *settings -> image paths -> video* +- **General** + - redesigned built-in profiler + - now includes both `python` and `torch` and traces individual functions + - use with `--debug --profile` + - **model merge** add **SD-XL ReBasin** support, thanks @AI-Casanova + - further UI optimizations for **mobile devices**, thanks @iDeNoh + - log level defaults to info for console and debug for log file + - better prompt display in process tab + - increase maximum lora cache values + - fix extra networks sorting + - fix controlnet compatibility issues in original backend + - fix img2img/inpaint paste params + - fix save text file for manually saved images + - fix python 3.9 compatibility issues + ## Update for 2023-11-23 New release, primarily focused around three major new features: full **LCM** support, completely new **Model Merge** functionality and **Stable-fast** compile support diff --git a/cli/hf-convert.py b/cli/hf-convert.py index 02599bf3e..eeefdce5e 100755 --- a/cli/hf-convert.py +++ b/cli/hf-convert.py @@ -8,7 +8,7 @@ import diffusers import safetensors import safetensors.torch as sf -log = logging.getLogger("sdnext") +log = logging.getLogger("sd") logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s | %(message)s') diff --git a/extensions-builtin/Lora/extra_networks_lora.py b/extensions-builtin/Lora/extra_networks_lora.py index 82aa82454..eb9a40b0d 100644 --- a/extensions-builtin/Lora/extra_networks_lora.py +++ b/extensions-builtin/Lora/extra_networks_lora.py @@ -64,8 +64,9 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): self.active = False def deactivate(self, p): - if shared.backend == shared.Backend.DIFFUSERS and hasattr(shared.sd_model, "unload_lora_weights"): - shared.sd_model.unload_lora_weights() + if shared.backend == shared.Backend.DIFFUSERS and hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"): + if 'CLIP' in shared.sd_model.text_encoder.__class__.__name__ and not (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"): + shared.sd_model.unload_lora_weights() if not self.active and getattr(networks, "originals", None ) is not None: networks.originals.undo() # remove patches if networks.debug: diff --git a/extensions-builtin/Lora/lora_patches.py b/extensions-builtin/Lora/lora_patches.py index 12986e8f9..983d80240 100644 --- a/extensions-builtin/Lora/lora_patches.py +++ b/extensions-builtin/Lora/lora_patches.py @@ -1,7 +1,10 @@ +import os import torch import networks from modules import patches, shared +# OpenVINO only works with Diffusers LoRa loading +force_lora_diffusers = os.environ.get('SD_LORA_DIFFUSERS', None) is not None class LoraPatches: def __init__(self): @@ -18,7 +21,7 @@ class LoraPatches: self.MultiheadAttention_load_state_dict = None def apply(self): - if self.active or (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"): # OpenVINO only works with Diffusers LoRa loading + if self.active or force_lora_diffusers: return self.Linear_forward = patches.patch(__name__, torch.nn.Linear, 'forward', networks.network_Linear_forward) self.Linear_load_state_dict = patches.patch(__name__, torch.nn.Linear, '_load_from_state_dict', networks.network_Linear_load_state_dict) @@ -36,7 +39,7 @@ class LoraPatches: self.active = True def undo(self): - if not self.active or (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"): # OpenVINO only works with Diffusers LoRa loading + if not self.active or force_lora_diffusers: return self.Linear_forward = patches.undo(__name__, torch.nn.Linear, 'forward') # pylint: disable=E1128 self.Linear_load_state_dict = patches.undo(__name__, torch.nn.Linear, '_load_from_state_dict') # pylint: disable=E1128 diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index b01c75c7a..40f19bde1 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -150,6 +150,10 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No recompile_model = True shared.compiled_model_state.lora_model = [] break + if not recompile_model: + if len(loaded_networks) > 0 and debug: + shared.log.debug('OpenVINO: Skipping LoRa loading') + return else: recompile_model = True shared.compiled_model_state.lora_model = [] @@ -166,11 +170,10 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No try: if recompile_model: shared.compiled_model_state.lora_model.append(f"{name}:{te_multipliers[i] if te_multipliers else 1.0}") - if shared.backend == shared.Backend.DIFFUSERS and (os.environ.get('SD_LORA_DIFFUSERS', None) - or getattr(network_on_disk, 'shorthash', None) == 'aaebf6360f7d' # lcm sd15 - or getattr(network_on_disk, 'shorthash', None) == '3d18b05e4f56' # lcm sdxl - or (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx")): - # OpenVINO only works with Diffusers LoRa loading. + if shared.backend == shared.Backend.DIFFUSERS and (os.environ.get('SD_LORA_DIFFUSERS', None) is not None): # OpenVINO only works with Diffusers LoRa loading. + # or getattr(network_on_disk, 'shorthash', '').lower() == 'aaebf6360f7d' # sd15-lcm + # or getattr(network_on_disk, 'shorthash', '').lower() == '3d18b05e4f56' # sdxl-lcm + # or getattr(network_on_disk, 'shorthash', '').lower() == '813ea5fb1c67' # turbo sdxl-turbo net = load_diffusers(name, network_on_disk, lora_scale=te_multipliers[i] if te_multipliers else 1.0) else: net = load_network(name, network_on_disk) diff --git a/extensions-builtin/Lora/scripts/lora_script.py b/extensions-builtin/Lora/scripts/lora_script.py index 49086f8cc..dc18866a9 100644 --- a/extensions-builtin/Lora/scripts/lora_script.py +++ b/extensions-builtin/Lora/scripts/lora_script.py @@ -1,22 +1,16 @@ import re -import gradio as gr from fastapi import FastAPI -import network import networks -import lora # noqa:F401 # pylint: disable=unused-import -# import lora_patches -import extra_networks_lora -import ui_extra_networks_lora +from network import NetworkOnDisk +from ui_extra_networks_lora import ExtraNetworksPageLora +from extra_networks_lora import ExtraNetworkLora +# import lora # noqa:F401 # pylint: disable=unused-import from modules import script_callbacks, ui_extra_networks, extra_networks, shared -# def unload(): -# networks.originals.undo() - - def before_ui(): - ui_extra_networks.register_page(ui_extra_networks_lora.ExtraNetworksPageLora()) - networks.extra_network_lora = extra_networks_lora.ExtraNetworkLora() + ui_extra_networks.register_page(ExtraNetworksPageLora()) + networks.extra_network_lora = ExtraNetworkLora() extra_networks.register_extra_network(networks.extra_network_lora) # extra_networks.register_extra_network_alias(networks.extra_network_lora, "lyco") @@ -28,15 +22,7 @@ script_callbacks.on_before_ui(before_ui) script_callbacks.on_infotext_pasted(networks.infotext_pasted) -shared.options_templates.update(shared.options_section(('extra_networks', "Extra Networks"), { - # "sd_lora": shared.OptionInfo("None", "Add network to prompt", gr.Dropdown, lambda: {"choices": ["None", *networks.available_networks], "visible": False}, refresh=networks.list_available_networks), - "sd_lora": shared.OptionInfo("None", "Add network to prompt", gr.Dropdown, {"choices": ["None"], "visible": False}), - # "lora_show_all": shared.OptionInfo(False, "Always show all networks on the Lora page").info("otherwise, those detected as for incompatible version of Stable Diffusion will be hidden"), - # "lora_hide_unknown_for_versions": shared.OptionInfo([], "Hide networks of unknown versions for model versions", gr.CheckboxGroup, {"choices": ["SD1", "SD2", "SDXL"]}), -})) - - -def create_lora_json(obj: network.NetworkOnDisk): +def create_lora_json(obj: NetworkOnDisk): return { "name": obj.name, "alias": obj.alias, @@ -45,7 +31,7 @@ def create_lora_json(obj: network.NetworkOnDisk): } -def api_networks(_: gr.Blocks, app: FastAPI): +def api_networks(_, app: FastAPI): @app.get("/sdapi/v1/loras") async def get_loras(): return [create_lora_json(obj) for obj in networks.available_networks.values()] diff --git a/extensions-builtin/Lora/ui_extra_networks_lora.py b/extensions-builtin/Lora/ui_extra_networks_lora.py index f029aaf05..bd408b327 100644 --- a/extensions-builtin/Lora/ui_extra_networks_lora.py +++ b/extensions-builtin/Lora/ui_extra_networks_lora.py @@ -91,7 +91,7 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): return None def list_items(self): - with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: future_items = {executor.submit(self.create_item, net): net for net in networks.available_networks} for future in concurrent.futures.as_completed(future_items): item = future.result() diff --git a/html/locale_en.json b/html/locale_en.json index 45074cb63..27d4beaa7 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -71,7 +71,7 @@ "extra networks": [ {"id":"","label":"UI position","localized":"","hint":"Location of extra networks"}, {"id":"","label":"cover","localized":"","hint":"cover full area"}, - {"id":"","label":"inline","localized":"","hint":"inline with all additional elelemtns (scrollable)"}, + {"id":"","label":"inline","localized":"","hint":"inline with all additional elements (scrollable)"}, {"id":"","label":"sidebar","localized":"","hint":"sidebar on the right side of the screen"}, {"id":"","label":"UI height (%)","localized":"","hint":""}, {"id":"","label":"UI sidebar width (%)","localized":"","hint":""}, @@ -445,7 +445,7 @@ {"id":"","label":"File format for grids","localized":"","hint":""}, {"id":"","label":"Add extended info (seed, prompt) to filename when saving grid","localized":"","hint":""}, {"id":"","label":"Grid row count","localized":"","hint":"Use -1 for autodetect and 0 for it to be same as batch size"}, - {"id":"","label":"Create text file next to every image with generation parameters","localized":"","hint":""}, + {"id":"","label":"Create info file for each every image","localized":"","hint":""}, {"id":"","label":"Create JSON log file for each saved image","localized":"","hint":"Save image information to a JSON file"}, {"id":"","label":"Save copy of image before doing face restoration","localized":"","hint":""}, {"id":"","label":"Save copy of image before applying hires","localized":"","hint":""}, diff --git a/html/reference.json b/html/reference.json index d1b2884dc..a3ff809ed 100644 --- a/html/reference.json +++ b/html/reference.json @@ -14,6 +14,26 @@ "desc": "Stable Diffusion XL (SDXL) is the latest AI image generation model that is tailored towards more photorealistic outputs with more detailed imagery and composition compared to previous SD models, including SD 2.1. It can make realistic faces, legible text within the images, and better image composition, all while using shorter and simpler prompts at a greatly increased base resolution of 1024x1024. Just like its predecessors, SDXL has the ability to generate image variations using image-to-image prompting, inpainting (reimagining of the selected parts of an image), and outpainting (creating new parts that lie outside the image borders).", "preview": "stabilityai--stable-diffusion-xl-base-1.0.jpg" }, + "StabilityAI SD 2.1 Turbo": { + "path": "stabilityai/sd-turbo", + "desc": "SD-Turbo is a distilled version of Stable Diffusion 2.1, trained for real-time synthesis. SD-Turbo is based on a novel training method called Adversarial Diffusion Distillation (ADD) (see the technical report), which allows sampling large-scale foundational image diffusion models in 1 to 4 steps at high image quality. This approach uses score distillation to leverage large-scale off-the-shelf image diffusion models as a teacher signal and combines this with an adversarial loss to ensure high image fidelity even in the low-step regime of one or two sampling steps.", + "preview": "stabilityai--sd-turbo.jpg" + }, + "StabilityAI SD-XL Turbo": { + "path": "stabilityai/sdxl-turbo", + "desc": "SDXL-Turbo is a distilled version of SDXL 1.0, trained for real-time synthesis. SDXL-Turbo is based on a novel training method called Adversarial Diffusion Distillation (ADD) (see the technical report), which allows sampling large-scale foundational image diffusion models in 1 to 4 steps at high image quality. This approach uses score distillation to leverage large-scale off-the-shelf image diffusion models as a teacher signal and combines this with an adversarial loss to ensure high image fidelity even in the low-step regime of one or two sampling steps.", + "preview": "stabilityai--sdxl-turbo.jpg" + }, + "StabilityAI Stable Video Diffusion": { + "path": "stabilityai/stable-video-diffusion-img2vid", + "desc": "(SVD) Image-to-Video is a latent diffusion model trained to generate short video clips from an image conditioning. This model was trained to generate 14 frames at resolution 576x1024 given a context frame of the same size. We also finetune the widely used f8-decoder for temporal consistency.", + "preview": "stabilityai--stable-video-diffusion-img2vid.jpg" + }, + "StabilityAI Stable Video Diffusion XT": { + "path": "stabilityai/stable-video-diffusion-img2vid-xt", + "desc": "(SVD) Image-to-Video is a latent diffusion model trained to generate short video clips from an image conditioning. This model was trained to generate 25 frames at resolution 576x1024 given a context frame of the same size, finetuned from SVD Image-to-Video [14 frames]. We also finetune the widely used f8-decoder for temporal consistency.", + "preview": "stabilityai--stable-video-diffusion-img2vid-xt.jpg" + }, "Segmind SSD-1B": { "path": "segmind/SSD-1B", "desc": "The Segmind Stable Diffusion Model (SSD-1B) offers a compact, efficient, and distilled version of the SDXL model. At 50% smaller and 60% faster than Stable Diffusion XL (SDXL), it provides quick and seamless performance without sacrificing image quality.", @@ -54,6 +74,11 @@ "desc": "Kandinsky 2.2 is a text-conditional diffusion model (+0.1!) based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder. Kandinsky 2.1 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas. It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.", "preview": "kandinsky-community--kandinsky-2-2-decoder.jpg" }, + "Kandinsky 3": { + "path": "kandinsky-community/kandinsky-3", + "desc": "Kandinsky 3.0 is an open-source text-to-image diffusion model built upon the Kandinsky2-x model family. In comparison to its predecessors, Kandinsky 3.0 incorporates more data and specifically related to Russian culture, which allows to generate pictures related to Russin culture. Furthermore, enhancements have been made to the text understanding and visual quality of the model, achieved by increasing the size of the text encoder and Diffusion U-Net models, respectively.", + "preview": "kandinsky-community--kandinsky-3.jpg" + }, "DeepFloyd IF Medium": { "path": "DeepFloyd/IF-I-M-v1.0", "desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.", diff --git a/installer.py b/installer.py index e5aaeceed..413463fbb 100644 --- a/installer.py +++ b/installer.py @@ -6,8 +6,6 @@ import shutil import logging import platform import subprocess -import io -import pstats import cProfile import pkg_resources @@ -29,6 +27,7 @@ opts = {} args = Dot({ 'debug': False, 'reset': False, + 'profile': False, 'upgrade': False, 'skip_extensions': False, 'skip_requirements': False, @@ -103,12 +102,6 @@ def setup_logging(): fh.doRollover() log_rolled = True - global first_call # pylint: disable=global-statement - if first_call: - log_size = os.path.getsize(log_file) if os.path.exists(log_file) else 0 - log.debug(f'Logger: file={log_file} level={level} size={log_size} mode={"append" if not log_rolled else "create"}') - first_call = False - fh.formatter = logging.Formatter('%(asctime)s | %(name)s | %(levelname)s | %(module)s | %(message)s') fh.setLevel(logging.DEBUG) log.addHandler(fh) @@ -121,10 +114,17 @@ def setup_logging(): # overrides logging.getLogger("urllib3").setLevel(logging.ERROR) logging.getLogger("httpx").setLevel(logging.ERROR) + logging.getLogger("diffusers").setLevel(logging.ERROR) + logging.getLogger("torch").setLevel(logging.ERROR) logging.getLogger("ControlNet").handlers = log.handlers logging.getLogger("lycoris").handlers = log.handlers # logging.getLogger("DeepSpeed").handlers = log.handlers +def get_logfile(): + log_size = os.path.getsize(log_file) if os.path.exists(log_file) else 0 + log.info(f'Logger: file="{log_file}" level={logging.getLevelName(logging.DEBUG if args.debug else logging.INFO)} size={log_size} mode={"append" if not log_rolled else "create"}') + return log_file + def custom_excepthook(exc_type, exc_value, exc_traceback): import traceback @@ -142,19 +142,9 @@ def print_dict(d): return ' '.join([f'{k}={v}' for k, v in d.items()]) -def print_profile(profile: cProfile.Profile, msg: str): - try: - from rich import print # pylint: disable=redefined-builtin - except Exception: - pass - profile.disable() - stream = io.StringIO() - ps = pstats.Stats(profile, stream=stream) - ps.sort_stats(pstats.SortKey.CUMULATIVE).print_stats(15) - profile = None - lines = stream.getvalue().split('\n') - lines = [line for line in lines if ' { // eslint-disable-line no-loop-func switch (sortVal) { - case 0: return a.dataset.name ? a.dataset.search.localeCompare(b.dataset.name) : 0; - case 1: return b.dataset.name ? b.dataset.search.localeCompare(a.dataset.name) : 0; + case 0: return a.dataset.search ? a.dataset.search.localeCompare(b.dataset.search) : 0; + case 1: return b.dataset.search ? b.dataset.search.localeCompare(a.dataset.search) : 0; case 2: return a.dataset.mtime && !isNaN(a.dataset.mtime) ? parseFloat(b.dataset.mtime) - parseFloat(a.dataset.mtime) : 0; case 3: return b.dataset.mtime && !isNaN(b.dataset.mtime) ? parseFloat(a.dataset.mtime) - parseFloat(b.dataset.mtime) : 0; case 4: return a.dataset.size && !isNaN(a.dataset.size) ? parseFloat(b.dataset.size) - parseFloat(a.dataset.size) : 0; diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 88bb60cc5..769d9da16 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -95,7 +95,7 @@ div#extras_scale_to_tab div.form{ flex-direction: row; } #img2img_sketch, #img2maskimg, #inpaint_sketch { overflow: overlay !important; resize: auto; background: var(--panel-background-fill); z-index: 5; } .image-buttons button{ min-width: auto; } .infotext { overflow-wrap: break-word; line-height: 1.5em; } -.infotext > p { padding-left: 1em; text-indent: -1em; } +.infotext > p { padding-left: 1em; text-indent: -1em; white-space: pre-wrap; } .tooltip { display: block; position: fixed; top: 1em; right: 1em; padding: 0.5em; background: var(--input-background-fill); color: var(--body-text-color); border: 1pt solid var(--button-primary-border-color); width: 22em; min-height: 1.3em; font-size: 0.8em; transition: opacity 0.2s ease-in; pointer-events: none; opacity: 0; z-index: 999; } .tooltip-show { opacity: 0.9; } @@ -184,7 +184,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt .extra-networks .description { flex: 3; } .extra-networks .tab-nav > button { margin-right: 0; height: 24px; padding: 2px 4px 2px 4px; } .extra-networks .buttons { position: absolute; right: 0; margin: -4px; background: var(--background-color); } -.extra-networks .buttons > button { margin-left: -0.4em; height: 1.4em; color: var(--primary-300) !important; } +.extra-networks .buttons > button { margin-left: -0.2em; height: 1.4em; color: var(--primary-300) !important; } .extra-networks .custom-button { width: 120px; width: 100%; background: none; justify-content: left; text-align: left; padding: 3px 3px 3px 12px; text-indent: -6px; box-shadow: none; line-break: auto; } .extra-networks .custom-button:hover { background: var(--button-primary-background-fill) } .extra-networks-tab { padding: 0 !important; } @@ -286,25 +286,25 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt /* Do not affect displays larger than 1024px wide. */ @media (max-width: 1024px) { - /* Screens smaller than 424px wide */ + /* Screens smaller than 400px wide */ @media (max-width: 399px) { :root, .light, .dark { --left-column: 100%; } /* maintain single column for from image operations on larger mobile devices */ #txt2img_results, #img2img_results, #extras_results { min-width: calc(min(320px, 100%)) !important;} + #txt2img_footer p { text-wrap: wrap; } + } - /* Screens larger than 425px wide */ - @media (min-width: 425px) { - :root, .light, .dark {--left-column: 50% ;} - - /* adjust extension panel to fit within resized sidebar */ - #scripts_alwayson_txt2img div { max-width: 99%; } + /* Screens larger than 400px wide */ + @media (min-width: 400px) { + :root, .light, .dark {--left-column: 50%;} /* maintain side by side split on larger mobile displays for from text */ - #txt2img_results, #extras_results { min-width: 50% !important;} + #txt2img_results, #extras_results, #txt2img_footer p {text-wrap: wrap; max-width: 100% !important; } } + #scripts_alwayson_txt2img div, #scripts_alwayson_img2img div { max-width: 100%; } #txt2img_prompt_container, #img2img_prompt_container { resize:vertical !important; } /* make generate and enqueue buttons take up the entire width of their rows. */ @@ -316,10 +316,10 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt #txt2img_generate_box, #img2img_generate_box, #txt2img_enqueue_wrapper,#img2img_enqueue_wrapper {display: flex;flex-direction: column;height: 4em !important;align-items: stretch;justify-content: space-evenly;} /* maintain single column for from image operations on larger mobile devices */ - #img2img_settings, #img2img_results { min-width: 100% !important; max-width: 100% !important;} + #img2img_interface, #img2img_results, #img2img_footer p {text-wrap: wrap; min-width: 100% !important; max-width: 100% !important;} /* fix inpaint image display being too large for mobile displays */ - #img2img_sketch, #img2maskimg, #inpaint_sketch {display: flex;alignment-baseline:after-edge !important;overflow: auto !important;resize: none !important;} - #img2maskimg canvas { width: 100% !important; max-height: 100% !important; height: auto !important; } + #img2img_sketch, #img2maskimg, #inpaint_sketch {display: flex; alignment-baseline:after-edge !important; overflow: auto !important; resize: none !important; } + #img2maskimg canvas { width: auto !important; max-height: 100% !important; height: auto !important; } /* fix from text/image UI elements to prevent them from moving around within the UI */ #txt2img_sampler, #txt2img_batch, #txt2img_seed_group, #txt2img_advanced, #txt2img_second_pass, #img2img_sampling_group, #img2img_resize_group, #img2img_batch_group, #img2img_seed_group, #img2img_denoise_group, #img2img_advanced_group { width: 100% !important; } diff --git a/launch.py b/launch.py index f47876240..a8c156b8c 100755 --- a/launch.py +++ b/launch.py @@ -4,7 +4,6 @@ import os import sys import time import shlex -import logging import subprocess from functools import lru_cache import installer @@ -171,6 +170,7 @@ if __name__ == "__main__": installer.args = args installer.setup_logging() installer.log.info('Starting SD.Next') + installer.get_logfile() try: sys.excepthook = installer.custom_excepthook except Exception: @@ -218,9 +218,6 @@ if __name__ == "__main__": installer.log.warning(f'See log file for more details: {installer.log_file}') installer.extensions_preload(parser) # adds additional args from extensions args = installer.parse_args(parser) - # installer.run_setup() - # installer.log.debug(f"Args: {vars(args)}") - logging.disable(logging.NOTSET if args.debug else logging.DEBUG) uv, instance = start_server(immediate=True, server=None) while True: diff --git a/models/Reference/stabilityai--sd-turbo.jpg b/models/Reference/stabilityai--sd-turbo.jpg new file mode 100644 index 000000000..875c42531 Binary files /dev/null and b/models/Reference/stabilityai--sd-turbo.jpg differ diff --git a/models/Reference/stabilityai--sdxl-turbo.jpg b/models/Reference/stabilityai--sdxl-turbo.jpg new file mode 100644 index 000000000..5a4609db1 Binary files /dev/null and b/models/Reference/stabilityai--sdxl-turbo.jpg differ diff --git a/models/Reference/stabilityai--stable-video-diffusion-img2vid-xt.jpg b/models/Reference/stabilityai--stable-video-diffusion-img2vid-xt.jpg new file mode 100644 index 000000000..81a809695 Binary files /dev/null and b/models/Reference/stabilityai--stable-video-diffusion-img2vid-xt.jpg differ diff --git a/models/Reference/stabilityai--stable-video-diffusion-img2vid.jpg b/models/Reference/stabilityai--stable-video-diffusion-img2vid.jpg new file mode 100644 index 000000000..81a809695 Binary files /dev/null and b/models/Reference/stabilityai--stable-video-diffusion-img2vid.jpg differ diff --git a/modules/call_queue.py b/modules/call_queue.py index 15d5274b8..21f94352d 100644 --- a/modules/call_queue.py +++ b/modules/call_queue.py @@ -2,9 +2,6 @@ import html import threading import time import cProfile -import pstats -import io -from rich import print # pylint: disable=redefined-builtin from modules import shared, progress, errors queue_lock = threading.Lock() @@ -62,10 +59,7 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False, name=None): else: res = list(res) if shared.cmd_opts.profile: - pr.disable() - s = io.StringIO() - pstats.Stats(pr, stream=s).sort_stats(pstats.SortKey.CUMULATIVE).print_stats(15) - print('Profile Exec:', s.getvalue()) + errors.profile(pr, 'Wrap') except Exception as e: errors.display(e, 'gradio call') if extra_outputs_array is None: diff --git a/modules/devices.py b/modules/devices.py index 78a20c4e6..0211242b5 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -150,6 +150,25 @@ def torch_gc(force=False): log.debug(f'gc: collected={collected} device={torch.device(get_optimal_device_name())} {memstats.memory_stats()}') +def set_cuda_sync_mode(mode): + """ + Set the CUDA device synchronization mode: auto, spin, yield or block. + auto: Chooses spin or yield depending on the number of available CPU cores. + spin: Runs one CPU core per GPU at 100% to poll for completed operations. + yield: Gives control to other threads between polling, if any are waiting. + block: Lets the thread sleep until the GPU driver signals completion. + """ + if mode == -1 or mode == 'none' or not cuda_ok: + return + try: + import ctypes + log.info(f'Set cuda synch: mode={mode}') + torch.cuda.set_device(torch.device(get_optimal_device_name())) + ctypes.CDLL('libcudart.so').cudaSetDeviceFlags({'auto': 0, 'spin': 1, 'yield': 2, 'block': 4}[mode]) + except Exception: + pass + + def test_fp16(): if shared.cmd_opts.experimental: return True @@ -272,6 +291,9 @@ dtype = torch.float16 dtype_vae = torch.float16 dtype_unet = torch.float16 unet_needs_upcast = False +if args.profile: + log.info(f'Torch build config: {torch.__config__.show()}') +# set_cuda_sync_mode('block') # none/auto/spin/yield/block def cond_cast_unet(tensor): diff --git a/modules/errors.py b/modules/errors.py index ecbc30a36..0f770856f 100644 --- a/modules/errors.py +++ b/modules/errors.py @@ -55,3 +55,41 @@ def run(code, task): def exception(suppress=[]): # noqa: B006 console.print_exception(show_locals=False, max_frames=10, extra_lines=2, suppress=suppress, theme="ansi_dark", word_wrap=False, width=min([console.width, 200])) + + +def profile(profiler, msg: str): + profiler.disable() + import io + import pstats + stream = io.StringIO() # pylint: disable=abstract-class-instantiated + p = pstats.Stats(profiler, stream=stream) + p.sort_stats(pstats.SortKey.CUMULATIVE) + p.print_stats(100) + # p.print_title() + # p.print_call_heading(10, 'time') + # p.print_callees(10) + # p.print_callers(10) + profiler = None + lines = stream.getvalue().split('\n') + lines = [x for x in lines if ' Dict if necessary - except: + except Exception: pass bake_in_vae_filename = sd_vae.vae_dict.get(kwargs.get("bake_in_vae", None), None) diff --git a/modules/images.py b/modules/images.py index 9c8def630..76c4a3cc6 100644 --- a/modules/images.py +++ b/modules/images.py @@ -244,6 +244,8 @@ def resize_image(resize_mode, im, width, height, upscaler_name=None, output_type if resize_mode == 0: res = im.copy() + if width == 0 or height == 0: + res = im.copy() elif resize_mode == 1: res = resize(im, width, height) elif resize_mode == 2: @@ -425,6 +427,20 @@ class FilenameGenerator: debug(f'Filename sanitize: input="{filename}" parts={parts} output="{fn}" ext={ext} max={max_length} len={len(fn)}') return fn + def sequence(self, x, dirname, basename): + if shared.opts.save_images_add_number or '[seq]' in x: + if '[seq]' not in x: + x = os.path.join(os.path.dirname(x), f"[seq]-{os.path.basename(x)}") + basecount = get_next_sequence_number(dirname, basename) + for i in range(9999): + seq = f"{basecount + i:05}" if basename == '' else f"{basename}-{basecount + i:04}" + filename = x.replace('[seq]', seq) + if not os.path.exists(filename): + debug(f'Prompt sequence: input="{x}" seq={seq} output="{filename}"') + x = filename + break + return x + def apply(self, x): res = '' for m in re_pattern.finditer(x): @@ -591,18 +607,7 @@ def save_image(image, path, basename='', seed=None, prompt=None, extension=share dirname = os.path.dirname(params.filename) if dirname is not None and len(dirname) > 0: os.makedirs(dirname, exist_ok=True) - # sequence - if shared.opts.save_images_add_number or '[seq]' in params.filename: - if '[seq]' not in params.filename: - params.filename = os.path.join(os.path.dirname(params.filename), f"[seq]-{os.path.basename(params.filename)}") - basecount = get_next_sequence_number(dirname, basename) - for i in range(9999): - seq = f"{basecount + i:05}" if basename == '' else f"{basename}-{basecount + i:04}" - filename = params.filename.replace('[seq]', seq) - if not os.path.exists(filename): - debug(f'Prompt sequence: input="{params.filename}" seq={seq} output="{filename}"') - params.filename = filename - break + params.filename = namegen.sequence(params.filename, dirname, basename) # callbacks script_callbacks.before_image_saved_callback(params) exifinfo = params.pnginfo.get('UserComment', '') @@ -618,6 +623,60 @@ def save_image(image, path, basename='', seed=None, prompt=None, extension=share return params.filename, filename_txt +def save_video_atomic(images, filename, video_type: str = 'none', duration: float = 2.0, loop: bool = False, interpolate: int = 0, scale: float = 1.0, pad: int = 1, change: float = 0.3): + try: + import cv2 + except Exception as e: + shared.log.error(f'Save video: cv2: {e}') + return + os.makedirs(os.path.dirname(filename), exist_ok=True) + if video_type.lower() == 'mp4': + frames = images + if interpolate > 0: + try: + import modules.rife + frames = modules.rife.interpolate(images, count=interpolate, scale=scale, pad=pad, change=change) + except Exception as e: + shared.log.error(f'RIFE interpolation: {e}') + errors.display(e, 'RIFE interpolation') + video_frames = [np.array(frame) for frame in frames] + fourcc = "mp4v" + h, w, _c = video_frames[0].shape + video_writer = cv2.VideoWriter(filename, fourcc=cv2.VideoWriter_fourcc(*fourcc), fps=len(frames)/duration, frameSize=(w, h)) + for i in range(len(video_frames)): + img = cv2.cvtColor(video_frames[i], cv2.COLOR_RGB2BGR) + video_writer.write(img) + shared.log.info(f'Save video: file="{filename}" frames={len(frames)} duration={duration} fourcc={fourcc}') + if video_type.lower() == 'gif' or video_type.lower() == 'png': + append = images.copy() + image = append.pop(0) + if loop: + append += append[::-1] + frames=len(append) + 1 + image.save( + filename, + save_all = True, + append_images = append, + optimize = False, + duration = 1000.0 * duration / frames, + loop = 0 if loop else 1, + ) + shared.log.info(f'Save video: file="{filename}" frames={len(append) + 1} duration={duration} loop={loop}') + + +def save_video(p, images, filename = None, video_type: str = 'none', duration: float = 2.0, loop: bool = False, interpolate: int = 0, scale: float = 1.0, pad: int = 1, change: float = 0.3): + if images is None or len(images) < 2 or video_type is None or video_type.lower() == 'none': + return + image = images[0] + namegen = FilenameGenerator(p, seed=p.all_seeds[0], prompt=p.all_prompts[0], image=image) + if filename is None: + filename = namegen.apply(shared.opts.samples_filename_pattern if shared.opts.samples_filename_pattern and len(shared.opts.samples_filename_pattern) > 0 else "[seq]-[prompt_words]") + filename = namegen.sanitize(os.path.join(shared.opts.outdir_video, filename)) + filename = namegen.sequence(filename, shared.opts.outdir_video, '') + filename = f'{filename}.{video_type.lower()}' + threading.Thread(target=save_video_atomic, args=(images, filename, video_type, duration, loop, interpolate, scale, pad, change)).start() + + def safe_decode_string(s: bytes): remove_prefix = lambda text, prefix: text[len(prefix):] if text.startswith(prefix) else text # pylint: disable=unnecessary-lambda-assignment for encoding in ['utf-8', 'utf-16', 'ascii', 'latin_1', 'cp1252', 'cp437']: # try different encodings diff --git a/modules/img2img.py b/modules/img2img.py index 0152cf9d5..e0fd159b0 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -208,7 +208,6 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s p.scale_by = scale_by p.scripts = modules.scripts.scripts_img2img p.script_args = args - p.extra_generation_params['Resize mode'] = resize_mode if mask: p.extra_generation_params["Mask blur"] = mask_blur p.extra_generation_params["Mask alpha"] = mask_alpha diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index 40c6551fe..cbadd14fc 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -31,6 +31,7 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.FloatTensor = torch.xpu.FloatTensor torch.Tensor.cuda = torch.Tensor.xpu torch.Tensor.is_cuda = torch.Tensor.is_xpu + torch.UntypedStorage.cuda = torch.UntypedStorage.xpu torch.cuda._initialization_lock = torch.xpu.lazy_init._initialization_lock torch.cuda._initialized = torch.xpu.lazy_init._initialized torch.cuda._lazy_seed_tracker = torch.xpu.lazy_init._lazy_seed_tracker diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index 432e805de..cdd83cea8 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -93,13 +93,25 @@ def linalg_solve(A, B, *args, **kwargs): # pylint: disable=invalid-name else: return original_linalg_solve(A, B, *args, **kwargs) +def is_cuda(self): + return self.device.type == 'xpu' + def ipex_hijacks(): + CondFunc('torch.tensor', + lambda orig_func, *args, device=None, **kwargs: orig_func(*args, device=return_xpu(device), **kwargs), + lambda orig_func, *args, device=None, **kwargs: check_device(device)) CondFunc('torch.Tensor.to', lambda orig_func, self, device=None, *args, **kwargs: orig_func(self, return_xpu(device), *args, **kwargs), lambda orig_func, self, device=None, *args, **kwargs: check_device(device)) CondFunc('torch.Tensor.cuda', lambda orig_func, self, device=None, *args, **kwargs: orig_func(self, return_xpu(device), *args, **kwargs), lambda orig_func, self, device=None, *args, **kwargs: check_device(device)) + CondFunc('torch.UntypedStorage.__init__', + lambda orig_func, *args, device=None, **kwargs: orig_func(*args, device=return_xpu(device), **kwargs), + lambda orig_func, *args, device=None, **kwargs: check_device(device)) + CondFunc('torch.UntypedStorage.cuda', + lambda orig_func, self, device=None, *args, **kwargs: orig_func(self, return_xpu(device), *args, **kwargs), + lambda orig_func, self, device=None, *args, **kwargs: check_device(device)) CondFunc('torch.empty', lambda orig_func, *args, device=None, **kwargs: orig_func(*args, device=return_xpu(device), **kwargs), lambda orig_func, *args, device=None, **kwargs: check_device(device)) @@ -112,9 +124,6 @@ def ipex_hijacks(): CondFunc('torch.zeros', lambda orig_func, *args, device=None, **kwargs: orig_func(*args, device=return_xpu(device), **kwargs), lambda orig_func, *args, device=None, **kwargs: check_device(device)) - CondFunc('torch.tensor', - lambda orig_func, *args, device=None, **kwargs: orig_func(*args, device=return_xpu(device), **kwargs), - lambda orig_func, *args, device=None, **kwargs: check_device(device)) CondFunc('torch.linspace', lambda orig_func, *args, device=None, **kwargs: orig_func(*args, device=return_xpu(device), **kwargs), lambda orig_func, *args, device=None, **kwargs: check_device(device)) @@ -124,7 +133,7 @@ def ipex_hijacks(): lambda orig_func, f, map_location=None, pickle_module=None, *, weights_only=False, mmap=None, **kwargs: check_device(map_location)) CondFunc('torch.Generator', - lambda orig_func, device=None: torch.xpu.Generator(device), + lambda orig_func, device=None: torch.xpu.Generator(return_xpu(device)), lambda orig_func, device=None: device is not None and device != torch.device("cpu") and device != "cpu") #TiledVAE and ControlNet: @@ -180,5 +189,6 @@ def ipex_hijacks(): torch.autocast = ipex_autocast torch.cat = torch_cat torch.linalg.solve = linalg_solve + torch.UntypedStorage.is_cuda = is_cuda torch.nn.functional.interpolate = interpolate torch.backends.cuda.sdp_kernel = return_null_context diff --git a/modules/loader.py b/modules/loader.py index 3dbb0c4a8..8b06f09ac 100644 --- a/modules/loader.py +++ b/modules/loader.py @@ -42,3 +42,16 @@ errors.install([gradio]) import diffusers # pylint: disable=W0611,C0411 timer.startup.record("diffusers") errors.log.info(f'Load packages: torch={getattr(torch, "__long_version__", torch.__version__)} diffusers={diffusers.__version__} gradio={gradio.__version__}') + +try: + import os + import math + cores = os.cpu_count() + affinity = len(os.sched_getaffinity(0)) + threads = torch.get_num_threads() + if threads < (affinity / 2): + torch.set_num_threads(math.floor(affinity / 2)) + threads = torch.get_num_threads() + errors.log.debug(f'Detected: cores={cores} affinity={affinity} set threads={threads}') +except Exception: + pass diff --git a/modules/lora b/modules/lora index 95ae56bd2..0908c5414 160000 --- a/modules/lora +++ b/modules/lora @@ -1 +1 @@ -Subproject commit 95ae56bd22c285ccb2fe5fca96d92f39842bb99b +Subproject commit 0908c5414dadfe2185669e2d657d542161647b79 diff --git a/modules/merging/merge.py b/modules/merging/merge.py index 4ea76fc2d..1f1ab55b4 100644 --- a/modules/merging/merge.py +++ b/modules/merging/merge.py @@ -13,10 +13,11 @@ from modules.merging import merge_methods from modules.merging.merge_utils import WeightClass from modules.merging.merge_rebasin import ( apply_permutation, - sdunet_permutation_spec, update_model_a, weight_matching, ) +from modules.merging.merge_PermSpec import sdunet_permutation_spec +from modules.merging.merge_PermSpec_SDXL import sdxl_permutation_spec ########################################################## # Files in modules.merging are heavily modified # versions of sd-meh by @s1dxl used with his blessing @@ -239,7 +240,10 @@ def rebasin_merge( ): # not sure how this does when 3 models are involved... model_a = thetas["model_a"].clone() - perm_spec = sdunet_permutation_spec() + if weight_matcher.SDXL: + perm_spec = sdxl_permutation_spec() + else: + perm_spec = sdunet_permutation_spec() for it in range(iterations): log_vram(f"rebasin: iteration={it}") @@ -319,7 +323,7 @@ def merge_key( # pylint: disable=inconsistent-return-statements for theta in thetas.values(): if key not in theta.keys(): - return + return thetas["model_a"][key] current_bases = weight_matcher(key) try: diff --git a/modules/merging/merge_PermSpec.py b/modules/merging/merge_PermSpec.py new file mode 100644 index 000000000..1de9b8317 --- /dev/null +++ b/modules/merging/merge_PermSpec.py @@ -0,0 +1,485 @@ +from modules.merging.merge_rebasin import PermutationSpec, permutation_spec_from_axes_to_perm +def sdunet_permutation_spec() -> PermutationSpec: + conv = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment + f"{name}.weight": ( + p_out, + p_in, + ), + f"{name}.bias": (p_out,), + } + norm = lambda name, p: {f"{name}.weight": (p,), f"{name}.bias": (p,)} # pylint: disable=unnecessary-lambda-assignment + dense = ( + lambda name, p_in, p_out, bias=True: { # pylint: disable=unnecessary-lambda-assignment + f"{name}.weight": (p_out, p_in), + f"{name}.bias": (p_out,), + } + if bias + else {f"{name}.weight": (p_out, p_in)} + ) + skip = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment + f"{name}": ( + p_out, + p_in, + None, + None, + ) + } + + # Unet Res blocks + easyblock = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment + **norm(f"{name}.in_layers.0", p_in), + **conv(f"{name}.in_layers.2", p_in, f"P_{name}_inner"), + **dense( + f"{name}.emb_layers.1", f"P_{name}_inner2", f"P_{name}_inner3", bias=True + ), + **norm(f"{name}.out_layers.0", f"P_{name}_inner4"), + **conv(f"{name}.out_layers.3", f"P_{name}_inner4", p_out), + } + + # VAE blocks - Unused + easyblock2 = lambda name, p: { # pylint: disable=unnecessary-lambda-assignment, unused-variable # noqa: F841 + **norm(f"{name}.norm1", p), + **conv(f"{name}.conv1", p, f"P_{name}_inner"), + **norm(f"{name}.norm2", f"P_{name}_inner"), + **conv(f"{name}.conv2", f"P_{name}_inner", p), + } + + # This is for blocks that use a residual connection, but change the number of channels via a Conv. + shortcutblock = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment, , unused-variable # noqa: F841 + **norm(f"{name}.norm1", p_in), + **conv(f"{name}.conv1", p_in, f"P_{name}_inner"), + **norm(f"{name}.norm2", f"P_{name}_inner"), + **conv(f"{name}.conv2", f"P_{name}_inner", p_out), + **conv(f"{name}.nin_shortcut", p_in, p_out), + **norm(f"{name}.nin_shortcut", p_out), + } + + return permutation_spec_from_axes_to_perm( + { + # Skipped Layers + **skip("betas", None, None), + **skip("alphas_cumprod", None, None), + **skip("alphas_cumprod_prev", None, None), + **skip("sqrt_alphas_cumprod", None, None), + **skip("sqrt_one_minus_alphas_cumprod", None, None), + **skip("log_one_minus_alphas_cumprods", None, None), + **skip("sqrt_recip_alphas_cumprod", None, None), + **skip("sqrt_recipm1_alphas_cumprod", None, None), + **skip("posterior_variance", None, None), + **skip("posterior_log_variance_clipped", None, None), + **skip("posterior_mean_coef1", None, None), + **skip("posterior_mean_coef2", None, None), + **skip("log_one_minus_alphas_cumprod", None, None), + **skip("model_ema.decay", None, None), + **skip("model_ema.num_updates", None, None), + # initial + **dense("model.diffusion_model.time_embed.0", None, "P_bg0", bias=True), + **dense("model.diffusion_model.time_embed.2", "P_bg0", "P_bg1", bias=True), + **conv("model.diffusion_model.input_blocks.0.0", "P_bg2", "P_bg3"), + # input blocks + **easyblock("model.diffusion_model.input_blocks.1.0", "P_bg4", "P_bg5"), + **norm("model.diffusion_model.input_blocks.1.1.norm", "P_bg6"), + **conv("model.diffusion_model.input_blocks.1.1.proj_in", "P_bg6", "P_bg7"), + **dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_q", "P_bg8", "P_bg9", bias=False), + **dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_k", "P_bg8", "P_bg9", bias=False), + **dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_v", "P_bg8", "P_bg9", bias=False), + **dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_out.0", "P_bg8", "P_bg9", bias=True), + **dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.ff.net.0.proj", "P_bg10", "P_bg11", bias=True), + **dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.ff.net.2", "P_bg12", "P_bg13", bias=True), + **dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_q", "P_bg14", "P_bg15", bias=False), + **dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_k", "P_bg16", "P_bg17", bias=False), + **dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_v", "P_bg16", "P_bg17", bias=False), + **dense("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_out.0", "P_bg18", "P_bg19", bias=True), + **norm("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.norm1", "P_bg19"), + **norm("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.norm2", "P_bg19"), + **norm("model.diffusion_model.input_blocks.1.1.transformer_blocks.0.norm3", "P_bg19"), + **conv("model.diffusion_model.input_blocks.1.1.proj_out", "P_bg19", "P_bg20"), + **easyblock("model.diffusion_model.input_blocks.2.0", "P_bg21", "P_bg22"), + **norm("model.diffusion_model.input_blocks.2.1.norm", "P_bg23"), + **conv("model.diffusion_model.input_blocks.2.1.proj_in", "P_bg23", "P_bg24"), + **dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn1.to_q", "P_bg25", "P_bg26", bias=False), + **dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn1.to_k", "P_bg25", "P_bg26", bias=False), + **dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn1.to_v", "P_bg25", "P_bg26", bias=False), + **dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn1.to_out.0", "P_bg25", "P_bg26", bias=True), + **dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.ff.net.0.proj", "P_bg27", "P_bg28", bias=True), + **dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.ff.net.2", "P_bg29", "P_bg30", bias=True), + **dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_q", "P_bg31", "P_bg32", bias=False), + **dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_k", "P_bg33", "P_bg34", bias=False), + **dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_v", "P_bg33", "P_bg34", bias=False), + **dense("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_out.0", "P_bg35", "P_bg36", bias=True), + **norm("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.norm1", "P_bg36"), + **norm("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.norm2", "P_bg36"), + **norm("model.diffusion_model.input_blocks.2.1.transformer_blocks.0.norm3", "P_bg36"), + **conv("model.diffusion_model.input_blocks.2.1.proj_out", "P_bg36", "P_bg37"), + **conv("model.diffusion_model.input_blocks.3.0.op", "P_bg38", "P_bg39"), + **easyblock("model.diffusion_model.input_blocks.4.0", "P_bg40", "P_bg41"), + **conv("model.diffusion_model.input_blocks.4.0.skip_connection", "P_bg42", "P_bg43"), + **norm("model.diffusion_model.input_blocks.4.1.norm", "P_bg44"), + **conv("model.diffusion_model.input_blocks.4.1.proj_in", "P_bg44", "P_bg45"), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_q", "P_bg46", "P_bg47", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_k", "P_bg46", "P_bg47", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_v", "P_bg46", "P_bg47", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_out.0", "P_bg46", "P_bg47", bias=True), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.ff.net.0.proj", "P_bg48", "P_bg49", bias=True), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.ff.net.2", "P_bg50", "P_bg51", bias=True), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_q", "P_bg52", "P_bg53", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_k", "P_bg54", "P_bg55", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_v", "P_bg54", "P_bg55", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_out.0", "P_bg56", "P_bg57", bias=True), + **norm("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.norm1", "P_bg57"), + **norm("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.norm2", "P_bg57"), + **norm("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.norm3", "P_bg57"), + **conv("model.diffusion_model.input_blocks.4.1.proj_out", "P_bg57", "P_bg58"), + **easyblock("model.diffusion_model.input_blocks.5.0", "P_bg59", "P_bg60"), + **norm("model.diffusion_model.input_blocks.5.1.norm", "P_bg61"), + **conv("model.diffusion_model.input_blocks.5.1.proj_in", "P_bg61", "P_bg62"), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_q", "P_bg63", "P_bg64", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_k", "P_bg63", "P_bg64", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_v", "P_bg63", "P_bg64", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_out.0", "P_bg63", "P_bg64", bias=True), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.ff.net.0.proj", "P_bg65", "P_bg66", bias=True), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.ff.net.2", "P_bg67", "P_bg68", bias=True), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_q", "P_bg69", "P_bg70", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_k", "P_bg71", "P_bg72", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_v", "P_bg71", "P_bg72", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_out.0", "P_bg73", "P_bg74", bias=True), + **norm("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.norm1", "P_bg74"), + **norm("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.norm2", "P_bg74"), + **norm("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.norm3", "P_bg74"), + **conv("model.diffusion_model.input_blocks.5.1.proj_out", "P_bg74", "P_bg75"), + **conv("model.diffusion_model.input_blocks.6.0.op", "P_bg76", "P_bg77"), + **easyblock("model.diffusion_model.input_blocks.7.0", "P_bg78", "P_bg79"), + **conv("model.diffusion_model.input_blocks.7.0.skip_connection", "P_bg80", "P_bg81"), + **norm("model.diffusion_model.input_blocks.7.1.norm", "P_bg82"), + **conv("model.diffusion_model.input_blocks.7.1.proj_in", "P_bg82", "P_bg83"), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_q", "P_bg84", "P_bg85", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_k", "P_bg84", "P_bg85", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_v", "P_bg84", "P_bg85", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_out.0", "P_bg84", "P_bg85", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.ff.net.0.proj", "P_bg86", "P_bg87", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.ff.net.2", "P_bg88", "P_bg89", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_q", "P_bg90", "P_bg91", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_k", "P_bg92", "P_bg93", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_v", "P_bg92", "P_bg93", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_out.0", "P_bg94", "P_bg95", bias=True), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.norm1", "P_bg95"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.norm2", "P_bg95"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.norm3", "P_bg95"), + **conv("model.diffusion_model.input_blocks.7.1.proj_out", "P_bg95", "P_bg96"), + **easyblock("model.diffusion_model.input_blocks.8.0", "P_bg97", "P_bg98"), + **norm("model.diffusion_model.input_blocks.8.1.norm", "P_bg99"), + **conv("model.diffusion_model.input_blocks.8.1.proj_in", "P_bg99", "P_bg100"), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_q", "P_bg101", "P_bg102", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_k", "P_bg101", "P_bg102", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_v", "P_bg101", "P_bg102", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_out.0", "P_bg101", "P_bg102", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.ff.net.0.proj", "P_bg103", "P_bg104", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.ff.net.2", "P_bg105", "P_bg106", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_q", "P_bg107", "P_bg108", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_k", "P_bg109", "P_bg110", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_v", "P_bg109", "P_bg110", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_out.0", "P_bg111", "P_bg112", bias=True), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.norm1", "P_bg112"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.norm2", "P_bg112"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.norm3", "P_bg112"), + **conv("model.diffusion_model.input_blocks.8.1.proj_out", "P_bg112", "P_bg113"), + **conv("model.diffusion_model.input_blocks.9.0.op", "P_bg114", "P_bg115"), + **easyblock("model.diffusion_model.input_blocks.10.0", "P_bg115", "P_bg116"), + **easyblock("model.diffusion_model.input_blocks.11.0", "P_bg116", "P_bg117"), + # middle blocks + **easyblock("model.diffusion_model.middle_block.0", "P_bg117", "P_bg118"), + **norm("model.diffusion_model.middle_block.1.norm", "P_bg119"), + **conv("model.diffusion_model.middle_block.1.proj_in", "P_bg119", "P_bg120"), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_q", "P_bg121", "P_bg122", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_k", "P_bg121", "P_bg122", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_v", "P_bg121", "P_bg122", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_out.0", "P_bg121", "P_bg122", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.ff.net.0.proj", "P_bg123", "P_bg124", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.ff.net.2", "P_bg125", "P_bg126", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_q", "P_bg127", "P_bg128", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_k", "P_bg129", "P_bg130", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_v", "P_bg129", "P_bg130", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_out.0", "P_bg131", "P_bg132", bias=True), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.0.norm1", "P_bg132"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.0.norm2", "P_bg132"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.0.norm3", "P_bg132"), + **conv("model.diffusion_model.middle_block.1.proj_out", "P_bg132", "P_bg133"), + **easyblock("model.diffusion_model.middle_block.2", "P_bg134", "P_bg135"), + # output blocks + **easyblock("model.diffusion_model.output_blocks.0.0", "P_bg136", "P_bg137"), + **conv("model.diffusion_model.output_blocks.0.0.skip_connection", "P_bg138", "P_bg139"), + **easyblock("model.diffusion_model.output_blocks.1.0", "P_bg140", "P_bg141"), + **conv("model.diffusion_model.output_blocks.1.0.skip_connection", "P_bg142", "P_bg143"), + **easyblock("model.diffusion_model.output_blocks.2.0", "P_bg144", "P_bg145"), + **conv("model.diffusion_model.output_blocks.2.0.skip_connection", "P_bg146", "P_bg147"), + **conv("model.diffusion_model.output_blocks.2.1.conv", "P_bg148", "P_bg149"), + **easyblock("model.diffusion_model.output_blocks.3.0", "P_bg150", "P_bg151"), + **conv("model.diffusion_model.output_blocks.3.0.skip_connection", "P_bg152", "P_bg153"), + **norm("model.diffusion_model.output_blocks.3.1.norm", "P_bg154"), + **conv("model.diffusion_model.output_blocks.3.1.proj_in", "P_bg154", "P_bg155"), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_q", "P_bg156", "P_bg157", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_k", "P_bg156", "P_bg157", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_v", "P_bg156", "P_bg157", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_out.0", "P_bg156", "P_bg157", bias=True), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.ff.net.0.proj", "P_bg158", "P_bg159", bias=True), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.ff.net.2", "P_bg160", "P_bg161", bias=True), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_q", "P_bg162", "P_bg163", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_k", "P_bg164", "P_bg165", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_v", "P_bg164", "P_bg165", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_out.0", "P_bg166", "P_bg167", bias=True), + **norm("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.norm1", "P_bg167"), + **norm("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.norm2", "P_bg167"), + **norm("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.norm3", "P_bg167"), + **conv("model.diffusion_model.output_blocks.3.1.proj_out", "P_bg167", "P_bg168"), + **easyblock("model.diffusion_model.output_blocks.4.0", "P_bg169", "P_bg170"), + **conv("model.diffusion_model.output_blocks.4.0.skip_connection", "P_bg171", "P_bg172"), + **norm("model.diffusion_model.output_blocks.4.1.norm", "P_bg173"), + **conv("model.diffusion_model.output_blocks.4.1.proj_in", "P_bg173", "P_bg174"), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_q", "P_bg175", "P_bg176", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_k", "P_bg175", "P_bg176", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_v", "P_bg175", "P_bg176", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_out.0", "P_bg175", "P_bg176", bias=True), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.ff.net.0.proj", "P_bg177", "P_bg178", bias=True), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.ff.net.2", "P_bg179", "P_bg180", bias=True), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_q", "P_bg181", "P_bg182", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_k", "P_bg183", "P_bg184", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_v", "P_bg183", "P_bg184", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_out.0", "P_bg185", "P_bg186", bias=True), + **norm("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.norm1", "P_bg186"), + **norm("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.norm2", "P_bg186"), + **norm("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.norm3", "P_bg186"), + **conv("model.diffusion_model.output_blocks.4.1.proj_out", "P_bg186", "P_bg187"), + **easyblock("model.diffusion_model.output_blocks.5.0", "P_bg188", "P_bg189"), + **conv("model.diffusion_model.output_blocks.5.0.skip_connection", "P_bg190", "P_bg191"), + **norm("model.diffusion_model.output_blocks.5.1.norm", "P_bg192"), + **conv("model.diffusion_model.output_blocks.5.1.proj_in", "P_bg192", "P_bg193"), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_q", "P_bg194", "P_bg195", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_k", "P_bg194", "P_bg195", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_v", "P_bg194", "P_bg195", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_out.0", "P_bg194", "P_bg195", bias=True), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.ff.net.0.proj", "P_bg196", "P_bg197", bias=True), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.ff.net.2", "P_bg198", "P_bg199", bias=True), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_q", "P_bg200", "P_bg201", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_k", "P_bg202", "P_bg203", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_v", "P_bg202", "P_bg203", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_out.0", "P_bg204", "P_bg205", bias=True), + **norm("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.norm1", "P_bg205"), + **norm("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.norm2", "P_bg205"), + **norm("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.norm3", "P_bg205"), + **conv("model.diffusion_model.output_blocks.5.1.proj_out", "P_bg205", "P_bg206"), + **conv("model.diffusion_model.output_blocks.5.2.conv", "P_bg206", "P_bg207"), + **easyblock("model.diffusion_model.output_blocks.6.0", "P_bg208", "P_bg209"), + **conv("model.diffusion_model.output_blocks.6.0.skip_connection", "P_bg210", "P_bg211"), + **norm("model.diffusion_model.output_blocks.6.1.norm", "P_bg212"), + **conv("model.diffusion_model.output_blocks.6.1.proj_in", "P_bg212", "P_bg213"), + **dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn1.to_q", "P_bg214", "P_bg215", bias=False), + **dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn1.to_k", "P_bg214", "P_bg215", bias=False), + **dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn1.to_v", "P_bg214", "P_bg215", bias=False), + **dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn1.to_out.0", "P_bg214", "P_bg215", bias=True), + **dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.ff.net.0.proj", "P_bg216", "P_bg217", bias=True), + **dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.ff.net.2", "P_bg218", "P_bg219", bias=True), + **dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn2.to_q", "P_bg220", "P_bg221", bias=False), + **dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn2.to_k", "P_bg222", "P_bg223", bias=False), + **dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn2.to_v", "P_bg222", "P_bg223", bias=False), + **dense("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn2.to_out.0", "P_bg224", "P_bg225", bias=True), + **norm("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.norm1", "P_bg225"), + **norm("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.norm2", "P_bg225"), + **norm("model.diffusion_model.output_blocks.6.1.transformer_blocks.0.norm3", "P_bg225"), + **conv("model.diffusion_model.output_blocks.6.1.proj_out", "P_bg225", "P_bg226"), + **easyblock("model.diffusion_model.output_blocks.7.0", "P_bg227", "P_bg228"), + **conv("model.diffusion_model.output_blocks.7.0.skip_connection", "P_bg229", "P_bg230"), + **norm("model.diffusion_model.output_blocks.7.1.norm", "P_bg231"), + **conv("model.diffusion_model.output_blocks.7.1.proj_in", "P_bg231", "P_bg232"), + **dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_q", "P_bg233", "P_bg234", bias=False), + **dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_k", "P_bg233", "P_bg234", bias=False), + **dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_v", "P_bg233", "P_bg234", bias=False), + **dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_out.0", "P_bg233", "P_bg234", bias=True), + **dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.ff.net.0.proj", "P_bg235", "P_bg236", bias=True), + **dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.ff.net.2", "P_bg237", "P_bg238", bias=True), + **dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn2.to_q", "P_bg239", "P_bg240", bias=False), + **dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn2.to_k", "P_bg241", "P_bg242", bias=False), + **dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn2.to_v", "P_bg241", "P_bg242", bias=False), + **dense("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn2.to_out.0", "P_bg243", "P_bg244", bias=True), + **norm("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.norm1", "P_bg244"), + **norm("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.norm2", "P_bg244"), + **norm("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.norm3", "P_bg244"), + **conv("model.diffusion_model.output_blocks.7.1.proj_out", "P_bg244", "P_bg245"), + **easyblock("model.diffusion_model.output_blocks.8.0", "P_bg246", "P_bg247"), + **conv("model.diffusion_model.output_blocks.8.0.skip_connection", "P_bg248", "P_bg249"), + **norm("model.diffusion_model.output_blocks.8.1.norm", "P_bg250"), + **conv("model.diffusion_model.output_blocks.8.1.proj_in", "P_bg250", "P_bg251"), + **dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn1.to_q", "P_bg252", "P_bg253", bias=False), + **dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn1.to_k", "P_bg252", "P_bg253", bias=False), + **dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn1.to_v", "P_bg252", "P_bg253", bias=False), + **dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn1.to_out.0", "P_bg252", "P_bg253", bias=True), + **dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.ff.net.0.proj", "P_bg254", "P_bg255", bias=True), + **dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.ff.net.2", "P_bg256", "P_bg257", bias=True), + **dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn2.to_q", "P_bg258", "P_bg259", bias=False), + **dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn2.to_k", "P_bg260", "P_bg261", bias=False), + **dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn2.to_v", "P_bg260", "P_bg261", bias=False), + **dense("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn2.to_out.0", "P_bg262", "P_bg263", bias=True), + **norm("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.norm1", "P_bg263"), + **norm("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.norm2", "P_bg263"), + **norm("model.diffusion_model.output_blocks.8.1.transformer_blocks.0.norm3", "P_bg263"), + **conv("model.diffusion_model.output_blocks.8.1.proj_out", "P_bg263", "P_bg264"), + **conv("model.diffusion_model.output_blocks.8.2.conv", "P_bg265", "P_bg266"), + **easyblock("model.diffusion_model.output_blocks.9.0", "P_bg267", "P_bg268"), + **conv("model.diffusion_model.output_blocks.9.0.skip_connection", "P_bg269", "P_bg270"), + **norm("model.diffusion_model.output_blocks.9.1.norm", "P_bg271"), + **conv("model.diffusion_model.output_blocks.9.1.proj_in", "P_bg271", "P_bg272"), + **dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn1.to_q", "P_bg273", "P_bg274", bias=False), + **dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn1.to_k", "P_bg273", "P_bg274", bias=False), + **dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn1.to_v", "P_bg273", "P_bg274", bias=False), + **dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn1.to_out.0", "P_bg273", "P_bg274", bias=True), + **dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.ff.net.0.proj", "P_bg275", "P_bg276", bias=True), + **dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.ff.net.2", "P_bg277", "P_bg278", bias=True), + **dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn2.to_q", "P_bg279", "P_bg280", bias=False), + **dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn2.to_k", "P_bg281", "P_bg282", bias=False), + **dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn2.to_v", "P_bg281", "P_bg282", bias=False), + **dense("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn2.to_out.0", "P_bg283", "P_bg284", bias=True), + **norm("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.norm1", "P_bg284"), + **norm("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.norm2", "P_bg284"), + **norm("model.diffusion_model.output_blocks.9.1.transformer_blocks.0.norm3", "P_bg284"), + **conv("model.diffusion_model.output_blocks.9.1.proj_out", "P_bg284", "P_bg285"), + **easyblock("model.diffusion_model.output_blocks.10.0", "P_bg286", "P_bg287"), + **conv("model.diffusion_model.output_blocks.10.0.skip_connection", "P_bg288", "P_bg289"), + **norm("model.diffusion_model.output_blocks.10.1.norm", "P_bg290"), + **conv("model.diffusion_model.output_blocks.10.1.proj_in", "P_bg290", "P_bg291"), + **dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn1.to_q", "P_bg292", "P_bg293", bias=False), + **dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn1.to_k", "P_bg292", "P_bg293", bias=False), + **dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn1.to_v", "P_bg292", "P_bg293", bias=False), + **dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn1.to_out.0", "P_bg292", "P_bg293", bias=True), + **dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.ff.net.0.proj", "P_b294", "P_bg295", bias=True), + **dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.ff.net.2", "P_bg296", "P_bg297", bias=True), + **dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn2.to_q", "P_bg298", "P_bg299", bias=False), + **dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn2.to_k", "P_bg300", "P_bg301", bias=False), + **dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn2.to_v", "P_bg300", "P_bg301", bias=False), + **dense("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn2.to_out.0", "P_bg302", "P_bg303", bias=True), + **norm("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.norm1", "P_bg303"), + **norm("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.norm2", "P_bg303"), + **norm("model.diffusion_model.output_blocks.10.1.transformer_blocks.0.norm3", "P_bg303"), + **conv("model.diffusion_model.output_blocks.10.1.proj_out", "P_bg303", "P_bg304"), + **easyblock("model.diffusion_model.output_blocks.11.0", "P_bg305", "P_bg306"), + **conv("model.diffusion_model.output_blocks.11.0.skip_connection", "P_bg307", "P_bg308"), + **norm("model.diffusion_model.output_blocks.11.1.norm", "P_bg309"), + **conv("model.diffusion_model.output_blocks.11.1.proj_in", "P_bg309", "P_bg310"), + **dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn1.to_q", "P_bg311", "P_bg312", bias=False), + **dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn1.to_k", "P_bg311", "P_bg312", bias=False), + **dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn1.to_v", "P_bg311", "P_bg312", bias=False), + **dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn1.to_out.0", "P_bg311", "P_bg312", bias=True), + **dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.ff.net.0.proj", "P_bg313", "P_bg314", bias=True), + **dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.ff.net.2", "P_bg315", "P_bg316", bias=True), + **dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn2.to_q", "P_bg317", "P_bg318", bias=False), + **dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn2.to_k", "P_bg319", "P_bg320", bias=False), + **dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn2.to_v", "P_bg319", "P_bg320", bias=False), + **dense("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn2.to_out.0", "P_bg321", "P_bg322", bias=True), + **norm("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.norm1", "P_bg322"), + **norm("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.norm2", "P_bg322"), + **norm("model.diffusion_model.output_blocks.11.1.transformer_blocks.0.norm3", "P_bg322"), + **conv("model.diffusion_model.output_blocks.11.1.proj_out", "P_bg322", "P_bg323"), + **norm("model.diffusion_model.out.0", "P_bg324"), + **conv("model.diffusion_model.out.2", "P_bg325", "P_bg326"), + **skip("cond_stage_model.transformer.text_model.embeddings.position_ids", None, None), + **dense("cond_stage_model.transformer.text_model.embeddings.token_embedding", "P_bg365", "P_bg366", bias=False), + **dense("cond_stage_model.transformer.text_model.embeddings.token_embedding", None, None), + **dense("cond_stage_model.transformer.text_model.embeddings.position_embedding", "P_bg367", "P_bg368", bias=False), + # cond stage text encoder + **dense("cond_stage_model.transformer.text_model.encoder.layers.0.self_attn.k_proj", "P_bg369", "P_bg370", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.0.self_attn.v_proj", "P_bg369", "P_bg370", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.0.self_attn.q_proj", "P_bg369", "P_bg370", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.0.self_attn.out_proj", "P_bg369", "P_bg370", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.0.layer_norm1", "P_bg370"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.0.mlp.fc1", "P_bg370", "P_bg371", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.0.mlp.fc2", "P_bg371", "P_bg372", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.0.layer_norm2", "P_bg372"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.1.self_attn.k_proj", "P_bg372", "P_bg373", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.1.self_attn.v_proj", "P_bg372", "P_bg373", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.1.self_attn.q_proj", "P_bg372", "P_bg373", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.1.self_attn.out_proj", "P_bg372", "P_bg373", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.1.layer_norm1", "P_bg373"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.1.mlp.fc1", "P_bg373", "P_bg374", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.1.mlp.fc2", "P_bg374", "P_bg375", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.1.layer_norm2", "P_bg375"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.2.self_attn.k_proj", "P_bg375", "P_bg376", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.2.self_attn.v_proj", "P_bg375", "P_bg376", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.2.self_attn.q_proj", "P_bg375", "P_bg376", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.2.self_attn.out_proj", "P_bg375", "P_bg376", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.2.layer_norm1", "P_bg376"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.2.mlp.fc1", "P_bg376", "P_bg377", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.2.mlp.fc2", "P_bg377", "P_bg378", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.2.layer_norm2", "P_bg378"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.3.self_attn.k_proj", "P_bg378", "P_bg379", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.3.self_attn.v_proj", "P_bg378", "P_bg379", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.3.self_attn.q_proj", "P_bg378", "P_bg379", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.3.self_attn.out_proj", "P_bg378", "P_bg379", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.3.layer_norm1", "P_bg379"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.3.mlp.fc1", "P_bg379", "P_bg380", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.3.mlp.fc2", "P_bg380", "P_b381", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.3.layer_norm2", "P_bg381"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.4.self_attn.k_proj", "P_bg381", "P_bg382", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.4.self_attn.v_proj", "P_bg381", "P_bg382", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.4.self_attn.q_proj", "P_bg381", "P_bg382", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.4.self_attn.out_proj", "P_bg381", "P_bg382", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.4.layer_norm1", "P_bg382"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.4.mlp.fc1", "P_bg382", "P_bg383", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.4.mlp.fc2", "P_bg383", "P_bg384", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.4.layer_norm2", "P_bg384"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.5.self_attn.k_proj", "P_bg384", "P_bg385", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.5.self_attn.v_proj", "P_bg384", "P_bg385", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.5.self_attn.q_proj", "P_bg384", "P_bg385", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.5.self_attn.out_proj", "P_bg384", "P_bg385", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.5.layer_norm1", "P_bg385"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.5.mlp.fc1", "P_bg385", "P_bg386", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.5.mlp.fc2", "P_bg386", "P_bg387", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.5.layer_norm2", "P_bg387"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.6.self_attn.k_proj", "P_bg387", "P_bg388", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.6.self_attn.v_proj", "P_bg387", "P_bg388", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.6.self_attn.q_proj", "P_bg387", "P_bg388", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.6.self_attn.out_proj", "P_bg387", "P_bg388", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.6.layer_norm1", "P_bg389"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.6.mlp.fc1", "P_bg389", "P_bg390", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.6.mlp.fc2", "P_bg390", "P_bg391", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.6.layer_norm2", "P_bg391"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.7.self_attn.k_proj", "P_bg391", "P_bg392", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.7.self_attn.v_proj", "P_bg391", "P_bg392", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.7.self_attn.q_proj", "P_bg391", "P_bg392", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.7.self_attn.out_proj", "P_bg391", "P_bg392", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.7.layer_norm1", "P_bg392"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.7.mlp.fc1", "P_bg392", "P_bg393", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.7.mlp.fc2", "P_bg393", "P_bg394", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.7.layer_norm2", "P_bg394"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.8.self_attn.k_proj", "P_bg394", "P_bg395", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.8.self_attn.v_proj", "P_bg394", "P_bg395", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.8.self_attn.q_proj", "P_bg394", "P_bg395", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.8.self_attn.out_proj", "P_bg394", "P_bg395", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.8.layer_norm1", "P_bg395"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.8.mlp.fc1", "P_bg395", "P_bg396", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.8.mlp.fc2", "P_bg396", "P_bg397", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.8.layer_norm2", "P_bg397"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.9.self_attn.k_proj", "P_bg397", "P_bg398", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.9.self_attn.v_proj", "P_bg397", "P_bg398", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.9.self_attn.q_proj", "P_bg397", "P_bg398", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.9.self_attn.out_proj", "P_bg397", "P_bg398", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.9.layer_norm1", "P_bg398"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.9.mlp.fc1", "P_bg398", "P_bg399", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.9.mlp.fc2", "P_bg400", "P_bg401", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.9.layer_norm2", "P_bg401"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.10.self_attn.k_proj", "P_bg401", "P_bg402", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.10.self_attn.v_proj", "P_bg401", "P_bg402", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.10.self_attn.q_proj", "P_bg401", "P_bg402", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.10.self_attn.out_proj", "P_bg401", "P_bg402", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.10.layer_norm1", "P_bg402"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.10.mlp.fc1", "P_bg402", "P_bg403", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.10.mlp.fc2", "P_bg403", "P_bg404", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.10.layer_norm2", "P_bg404"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.11.self_attn.k_proj", "P_bg404", "P_bg405", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.11.self_attn.v_proj", "P_bg404", "P_bg405", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.11.self_attn.q_proj", "P_bg404", "P_bg405", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.11.self_attn.out_proj", "P_bg404", "P_bg405", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.11.layer_norm1", "P_bg405"), + **dense("cond_stage_model.transformer.text_model.encoder.layers.11.mlp.fc1", "P_bg405", "P_bg406", bias=True), + **dense("cond_stage_model.transformer.text_model.encoder.layers.11.mlp.fc2", "P_bg406", "P_bg407", bias=True), + **norm("cond_stage_model.transformer.text_model.encoder.layers.11.layer_norm2", "P_bg407"), + **norm("cond_stage_model.transformer.text_model.final_layer_norm", "P_bg407"), + } + ) diff --git a/modules/merging/merge_PermSpec_SDXL.py b/modules/merging/merge_PermSpec_SDXL.py new file mode 100644 index 000000000..2f1c6f8ef --- /dev/null +++ b/modules/merging/merge_PermSpec_SDXL.py @@ -0,0 +1,1860 @@ +from modules.merging.merge_rebasin import PermutationSpec, permutation_spec_from_axes_to_perm + +def sdxl_permutation_spec() -> PermutationSpec: + conv = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment + f"{name}.weight": ( + p_out, + p_in, + ), + f"{name}.bias": (p_out,), + } + norm = lambda name, p: {f"{name}.weight": (p,), f"{name}.bias": (p,)} # pylint: disable=unnecessary-lambda-assignment + dense = ( + lambda name, p_in, p_out, bias=True: { # pylint: disable=unnecessary-lambda-assignment + f"{name}.weight": (p_out, p_in), + f"{name}.bias": (p_out,), + } + if bias + else {f"{name}.weight": (p_out, p_in)} + ) + skip = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment + f"{name}": ( + p_out, + p_in, + None, + None, + ) + } + + # Unet Res blocks + easyblock = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment + **norm(f"{name}.in_layers.0", p_in), + **conv(f"{name}.in_layers.2", p_in, f"P_{name}_inner"), + **dense( + f"{name}.emb_layers.1", f"P_{name}_inner2", f"P_{name}_inner3", bias=True + ), + **norm(f"{name}.out_layers.0", f"P_{name}_inner4"), + **conv(f"{name}.out_layers.3", f"P_{name}_inner4", p_out), + } + + return permutation_spec_from_axes_to_perm( + { + # Skipped Layers + **skip("betas", None, None), + **skip("alphas_cumprod", None, None), + **skip("alphas_cumprod_prev", None, None), + **skip("sqrt_alphas_cumprod", None, None), + **skip("sqrt_one_minus_alphas_cumprod", None, None), + **skip("log_one_minus_alphas_cumprods", None, None), + **skip("sqrt_recip_alphas_cumprod", None, None), + **skip("sqrt_recipm1_alphas_cumprod", None, None), + **skip("posterior_variance", None, None), + **skip("posterior_log_variance_clipped", None, None), + **skip("posterior_mean_coef1", None, None), + **skip("posterior_mean_coef2", None, None), + **skip("log_one_minus_alphas_cumprod", None, None), + **skip("model_ema.decay", None, None), + **skip("model_ema.num_updates", None, None), + **skip("conditioner.embedders.0.transformer.text_model.embeddings.position_ids", None, None), + **skip("conditioner.embedders.1.model.logit_scale", None, None), + **skip("conditioner.embedders.1.model.positional_embedding", None, None), + **skip("conditioner.embedders.1.model.text_projection", None, None), + **conv("model.diffusion_model.input_blocks.0.0", "P_bg0", "P_bg1"), + **easyblock("model.diffusion_model.input_blocks.1.0", "P_bg2", "P_bg3"), + **easyblock("model.diffusion_model.input_blocks.2.0", "P_bg4", "P_bg5"), + **conv("model.diffusion_model.input_blocks.3.0.op", "P_bg6", "P_bg7"), + **easyblock("model.diffusion_model.input_blocks.4.0", "P_bg8", "P_bg9"), + **conv("model.diffusion_model.input_blocks.4.0.skip_connection", "P_bg10", "P_bg11"), + **norm("model.diffusion_model.input_blocks.4.1.norm", "P_bg12"), + **conv("model.diffusion_model.input_blocks.4.1.proj_in", "P_bg12", "P_bg13"), + **conv("model.diffusion_model.input_blocks.4.1.proj_out", "P_bg14", "P_bg15"), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_k", "P_bg16", "P_bg17", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_out.0", "P_bg16", "P_bg17", bias=True), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_q", "P_bg16", "P_bg17", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_v", "P_bg16", "P_bg17", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_k", "P_bg18", "P_bg19", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_out.0", "P_bg20", "P_bg21", bias=True), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_q", "P_bg20", "P_bg21", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_v", "P_bg18", "P_bg19", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.ff.net.0.proj", "P_bg22", "P_bg23", bias=True), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.ff.net.2", "P_bg24", "P_bg25", bias=True), + **norm("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.norm1", "P_bg26"), + **norm("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.norm2", "P_bg26"), + **norm("model.diffusion_model.input_blocks.4.1.transformer_blocks.0.norm3", "P_bg26"), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.attn1.to_k", "P_bg27", "P_bg28", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.attn1.to_out.0", "P_bg27", "P_bg28", bias=True), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.attn1.to_q", "P_bg27", "P_bg28", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.attn1.to_v", "P_bg27", "P_bg28", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.attn2.to_k", "P_bg29", "P_bg30", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.attn2.to_out.0", "P_bg31", "P_bg32", bias=True), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.attn2.to_q", "P_bg31", "P_bg32", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.attn2.to_v", "P_bg33", "P_bg34", bias=False), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.ff.net.0.proj", "P_bg35", "P_bg36", bias=True), + **dense("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.ff.net.2", "P_bg37", "P_bg38", bias=True), + **norm("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.norm1", "P_bg39"), + **norm("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.norm2", "P_bg39"), + **norm("model.diffusion_model.input_blocks.4.1.transformer_blocks.1.norm3", "P_bg39"), + **easyblock("model.diffusion_model.input_blocks.5.0", "P_bg40", "P_bg41"), + **norm("model.diffusion_model.input_blocks.5.1.norm", "P_bg42"), + **conv("model.diffusion_model.input_blocks.5.1.proj_in", "P_bg43", "P_bg44"), + **conv("model.diffusion_model.input_blocks.5.1.proj_out", "P_bg45", "P_bg46"), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_k", "P_bg47", "P_bg48", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_out.0", "P_bg47", "P_bg48", bias=True), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_q", "P_bg47", "P_bg48", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_v", "P_bg47", "P_bg48", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_k", "P_bg49", "P_bg50", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_out.0", "P_bg51", "P_bg52", bias=True), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_q", "P_bg53", "P_bg54", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_v", "P_bg55", "P_bg56", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.ff.net.0.proj", "P_bg57", "P_bg58", bias=True), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.ff.net.2", "P_bg59", "P_bg60", bias=True), + **norm("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.norm1", "P_bg61"), + **norm("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.norm2", "P_bg61"), + **norm("model.diffusion_model.input_blocks.5.1.transformer_blocks.0.norm3", "P_bg61"), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.attn1.to_k", "P_bg62", "P_bg63", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.attn1.to_out.0", "P_bg62", "P_bg63", bias=True), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.attn1.to_q", "P_bg62", "P_bg63", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.attn1.to_v", "P_bg62", "P_bg63", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.attn2.to_k", "P_bg64", "P_bg65", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.attn2.to_out.0", "P_bg66", "P_bg67", bias=True), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.attn2.to_q", "P_bg68", "P_bg69", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.attn2.to_v", "P_bg70", "P_bg71", bias=False), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.ff.net.0.proj", "P_bg72", "P_bg73", bias=True), + **dense("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.ff.net.2", "P_bg74", "P_bg75", bias=True), + **norm("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.norm1", "P_bg76"), + **norm("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.norm2", "P_bg76"), + **norm("model.diffusion_model.input_blocks.5.1.transformer_blocks.1.norm3", "P_bg76"), + **conv("model.diffusion_model.input_blocks.6.0.op", "P_bg77", "P_bg78"), + **easyblock("model.diffusion_model.input_blocks.7.0", "P_bg78", "P_bg79"), + **conv("model.diffusion_model.input_blocks.7.0.skip_connection", "P_bg80", "P_bg81"), + **norm("model.diffusion_model.input_blocks.7.1.norm", "P_bg82"), + **conv("model.diffusion_model.input_blocks.7.1.proj_in", "P_bg82", "P_bg83"), + **conv("model.diffusion_model.input_blocks.7.1.proj_out", "P_bg84", "P_bg85"), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_k", "P_bg86", "P_bg87", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_out.0", "P_bg86", "P_bg87", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_q", "P_bg86", "P_bg87", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_v", "P_bg86", "P_bg87", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_k", "P_bg88", "P_bg89", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_out.0", "P_bg90", "P_bg91", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_q", "P_bg92", "P_bg93", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_v", "P_bg94", "P_bg95", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.ff.net.0.proj", "P_bg96", "P_bg97", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.ff.net.2", "P_bg98", "P_bg99", bias=True), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.norm1", "P_bg100"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.norm2", "P_bg100"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.0.norm3", "P_bg100"), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.attn1.to_k", "P_bg101", "P_bg102", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.attn1.to_out.0", "P_bg101", "P_bg102", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.attn1.to_q", "P_bg101", "P_bg102", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.attn1.to_v", "P_bg101", "P_bg102", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.attn2.to_k", "P_bg103", "P_bg104", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.attn2.to_out.0", "P_bg105", "P_bg106", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.attn2.to_q", "P_bg107", "P_bg108", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.attn2.to_v", "P_bg109", "P_bg110", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.ff.net.0.proj", "P_bg111", "P_bg112", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.ff.net.2", "P_bg113", "P_bg114", bias=True), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.norm1", "P_bg115"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.norm2", "P_bg115"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.1.norm3", "P_bg115"), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.attn1.to_k", "P_bg116", "P_bg117", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.attn1.to_out.0", "P_bg116", "P_bg117", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.attn1.to_q", "P_bg116", "P_bg117", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.attn1.to_v", "P_bg116", "P_bg117", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.attn2.to_k", "P_bg118", "P_bg119", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.attn2.to_out.0", "P_bg120", "P_bg121", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.attn2.to_q", "P_bg122", "P_bg123", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.attn2.to_v", "P_bg124", "P_bg125", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.ff.net.0.proj", "P_bg126", "P_bg127", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.ff.net.2", "P_bg128", "P_bg129", bias=True), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.norm1", "P_bg130"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.norm2", "P_bg130"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.2.norm3", "P_bg130"), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.attn1.to_k", "P_bg110", "P_bg111", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.attn1.to_out.0", "P_bg131", "P_bg132", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.attn1.to_q", "P_bg131", "P_bg132", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.attn1.to_v", "P_bg131", "P_bg132", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.attn2.to_k", "P_bg133", "P_bg134", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.attn2.to_out.0", "P_bg135", "P_bg136", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.attn2.to_q", "P_bg137", "P_bg138", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.attn2.to_v", "P_bg139", "P_bg140", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.ff.net.0.proj", "P_bg141", "P_bg142", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.ff.net.2", "P_bg143", "P_bg144", bias=True), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.norm1", "P_bg145"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.norm2", "P_bg145"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.3.norm3", "P_bg145"), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.attn1.to_k", "P_bg146", "P_bg147", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.attn1.to_out.0", "P_bg146", "P_bg147", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.attn1.to_q", "P_bg146", "P_bg147", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.attn1.to_v", "P_bg146", "P_bg147", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.attn2.to_k", "P_bg148", "P_bg149", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.attn2.to_out.0", "P_bg150", "P_bg151", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.attn2.to_q", "P_bg152", "P_bg153", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.attn2.to_v", "P_bg154", "P_bg155", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.ff.net.0.proj", "P_bg156", "P_bg157", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.ff.net.2", "P_bg158", "P_bg159", bias=True), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.norm1", "P_bg160"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.norm2", "P_bg160"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.4.norm3", "P_bg160"), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.attn1.to_k", "P_bg161", "P_bg162", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.attn1.to_out.0", "P_bg161", "P_bg162", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.attn1.to_q", "P_bg161", "P_bg162", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.attn1.to_v", "P_bg161", "P_bg162", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.attn2.to_k", "P_bg163", "P_bg164", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.attn2.to_out.0", "P_bg165", "P_bg166", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.attn2.to_q", "P_bg167", "P_bg168", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.attn2.to_v", "P_bg169", "P_bg170", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.ff.net.0.proj", "P_bg171", "P_bg172", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.ff.net.2", "P_bg173", "P_bg174", bias=True), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.norm1", "P_bg175"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.norm2", "P_bg175"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.5.norm3", "P_bg175"), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.attn1.to_k", "P_bg176", "P_bg177", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.attn1.to_out.0", "P_bg176", "P_bg177", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.attn1.to_q", "P_bg176", "P_bg177", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.attn1.to_v", "P_bg176", "P_bg177", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.attn2.to_k", "P_bg178", "P_bg179", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.attn2.to_out.0", "P_bg180", "P_bg181", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.attn2.to_q", "P_bg182", "P_bg183", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.attn2.to_v", "P_bg184", "P_bg185", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.ff.net.0.proj", "P_bg186", "P_bg187", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.ff.net.2", "P_bg188", "P_bg189", bias=True), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.norm1", "P_bg190"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.norm2", "P_bg190"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.6.norm3", "P_bg190"), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.attn1.to_k", "P_bg191", "P_bg192", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.attn1.to_out.0", "P_bg193", "P_bg194", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.attn1.to_q", "P_bg195", "P_bg196", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.attn1.to_v", "P_bg197", "P_bg198", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.attn2.to_k", "P_bg199", "P_bg200", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.attn2.to_out.0", "P_bg201", "P_bg202", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.attn2.to_q", "P_bg203", "P_bg204", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.attn2.to_v", "P_bg205", "P_bg206", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.ff.net.0.proj", "P_bg207", "P_bg208", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.ff.net.2", "P_bg209", "P_bg210", bias=True), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.norm1", "P_bg211"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.norm2", "P_bg211"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.7.norm3", "P_bg211"), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.attn1.to_k", "P_bg212", "P_bg213", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.attn1.to_out.0", "P_bg212", "P_bg213", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.attn1.to_q", "P_bg212", "P_bg213", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.attn1.to_v", "P_bg212", "P_bg213", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.attn2.to_k", "P_bg214", "P_bg215", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.attn2.to_out.0", "P_bg216", "P_bg217", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.attn2.to_q", "P_bg218", "P_bg219", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.attn2.to_v", "P_bg220", "P_bg221", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.ff.net.0.proj", "P_bg222", "P_bg223", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.ff.net.2", "P_bg224", "P_bg225", bias=True), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.norm1", "P_bg226"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.norm2", "P_bg226"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.8.norm3", "P_bg226"), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.attn1.to_k", "P_bg227", "P_bg228", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.attn1.to_out.0", "P_bg227", "P_bg228", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.attn1.to_q", "P_bg227", "P_bg228", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.attn1.to_v", "P_bg227", "P_bg228", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.attn2.to_k", "P_bg229", "P_bg230", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.attn2.to_out.0", "P_bg231", "P_bg232", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.attn2.to_q", "P_bg233", "P_bg234", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.attn2.to_v", "P_bg235", "P_bg236", bias=False), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.ff.net.0.proj", "P_bg237", "P_bg238", bias=True), + **dense("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.ff.net.2", "P_bg239", "P_bg240", bias=True), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.norm1", "P_bg241"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.norm2", "P_bg241"), + **norm("model.diffusion_model.input_blocks.7.1.transformer_blocks.9.norm3", "P_bg241"), + **easyblock("model.diffusion_model.input_blocks.8.0", "P_bg242", "P_bg243"), + **norm("model.diffusion_model.input_blocks.8.1.norm", "P_bg244"), + **conv("model.diffusion_model.input_blocks.8.1.proj_in", "P_bg244", "P_bg245"), + **conv("model.diffusion_model.input_blocks.8.1.proj_out", "P_bg246", "P_bg247"), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_k", "P_bg248", "P_bg249", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_out.0", "P_bg248", "P_bg249", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_q", "P_bg248", "P_bg249", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_v", "P_bg248", "P_bg249", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_k", "P_bg250", "P_bg251", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_out.0", "P_bg252", "P_bg253", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_q", "P_bg254", "P_bg255", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_v", "P_bg256", "P_bg257", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.ff.net.0.proj", "P_bg258", "P_bg259", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.ff.net.2", "P_bg260", "P_bg261", bias=True), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.norm1", "P_bg262"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.norm2", "P_bg262"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.0.norm3", "P_bg262"), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.attn1.to_k", "P_bg263", "P_bg264", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.attn1.to_out.0", "P_bg263", "P_bg264", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.attn1.to_q", "P_bg263", "P_bg264", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.attn1.to_v", "P_bg263", "P_bg264", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.attn2.to_k", "P_bg265", "P_bg266", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.attn2.to_out.0", "P_bg267", "P_bg268", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.attn2.to_q", "P_bg269", "P_bg270", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.attn2.to_v", "P_bg271", "P_bg272", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.ff.net.0.proj", "P_bg273", "P_bg274", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.ff.net.2", "P_bg275", "P_bg276", bias=True), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.norm1", "P_bg277"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.norm2", "P_bg277"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.1.norm3", "P_bg277"), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.attn1.to_k", "P_bg278", "P_bg279", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.attn1.to_out.0", "P_bg278", "P_bg279", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.attn1.to_q", "P_bg278", "P_bg279", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.attn1.to_v", "P_bg278", "P_bg279", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.attn2.to_k", "P_bg280", "P_bg281", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.attn2.to_out.0", "P_bg282", "P_bg283", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.attn2.to_q", "P_bg284", "P_bg285", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.attn2.to_v", "P_bg286", "P_bg287", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.ff.net.0.proj", "P_bg288", "P_bg289", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.ff.net.2", "P_bg290", "P_bg291", bias=True), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.norm1", "P_bg292"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.norm2", "P_bg292"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.2.norm3", "P_bg292"), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.attn1.to_k", "P_bg293", "P_bg294", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.attn1.to_out.0", "P_bg293", "P_bg294", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.attn1.to_q", "P_bg293", "P_bg294", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.attn1.to_v", "P_bg293", "P_bg294", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.attn2.to_k", "P_bg295", "P_bg296", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.attn2.to_out.0", "P_bg297", "P_bg298", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.attn2.to_q", "P_bg299", "P_bg300", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.attn2.to_v", "P_bg301", "P_bg302", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.ff.net.0.proj", "P_bg303", "P_bg304", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.ff.net.2", "P_bg305", "P_bg306", bias=True), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.norm1", "P_bg307"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.norm2", "P_bg307"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.3.norm3", "P_bg307"), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.attn1.to_k", "P_bg308", "P_bg309", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.attn1.to_out.0", "P_bg308", "P_bg309", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.attn1.to_q", "P_bg308", "P_bg309", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.attn1.to_v", "P_bg308", "P_bg309", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.attn2.to_k", "P_bg310", "P_bg311", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.attn2.to_out.0", "P_bg312", "P_bg313", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.attn2.to_q", "P_bg314", "P_bg315", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.attn2.to_v", "P_bg316", "P_bg317", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.ff.net.0.proj", "P_bg318", "P_bg319", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.ff.net.2", "P_bg320", "P_bg321", bias=True), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.norm1", "P_bg322"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.norm2", "P_bg322"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.4.norm3", "P_bg322"), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.attn1.to_k", "P_bg322", "P_bg323", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.attn1.to_out.0", "P_bg322", "P_bg323", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.attn1.to_q", "P_bg322", "P_bg323", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.attn1.to_v", "P_bg322", "P_bg323", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.attn2.to_k", "P_bg324", "P_bg325", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.attn2.to_out.0", "P_bg326", "P_bg327", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.attn2.to_q", "P_bg328", "P_bg329", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.attn2.to_v", "P_bg330", "P_bg331", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.ff.net.0.proj", "P_bg332", "P_bg333", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.ff.net.2", "P_bg334", "P_bg335", bias=True), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.norm1", "P_bg336"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.norm2", "P_bg336"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.5.norm3", "P_bg336"), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.attn1.to_k", "P_bg337", "P_bg338", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.attn1.to_out.0", "P_bg337", "P_bg338", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.attn1.to_q", "P_bg337", "P_bg338", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.attn1.to_v", "P_bg337", "P_bg338", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.attn2.to_k", "P_bg339", "P_bg340", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.attn2.to_out.0", "P_bg341", "P_bg342", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.attn2.to_q", "P_bg343", "P_bg344", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.attn2.to_v", "P_bg345", "P_bg346", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.ff.net.0.proj", "P_bg347", "P_bg348", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.ff.net.2", "P_bg349", "P_bg350", bias=True), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.norm1", "P_bg351"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.norm2", "P_bg351"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.6.norm3", "P_bg351"), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.attn1.to_k", "P_bg352", "P_bg353", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.attn1.to_out.0", "P_bg352", "P_bg353", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.attn1.to_q", "P_bg352", "P_bg353", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.attn1.to_v", "P_bg352", "P_bg353", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.attn2.to_k", "P_bg354", "P_bg355", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.attn2.to_out.0", "P_bg356", "P_bg357", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.attn2.to_q", "P_bg358", "P_bg359", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.attn2.to_v", "P_bg360", "P_bg361", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.ff.net.0.proj", "P_bg362", "P_bg363", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.ff.net.2", "P_bg364", "P_bg365", bias=True), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.norm1", "P_bg366"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.norm2", "P_bg366"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.7.norm3", "P_bg366"), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.attn1.to_k", "P_bg367", "P_bg368", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.attn1.to_out.0", "P_bg367", "P_bg368", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.attn1.to_q", "P_bg367", "P_bg368", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.attn1.to_v", "P_bg367", "P_bg368", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.attn2.to_k", "P_bg369", "P_bg370", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.attn2.to_out.0", "P_bg371", "P_bg372", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.attn2.to_q", "P_bg373", "P_bg374", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.attn2.to_v", "P_bg375", "P_bg376", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.ff.net.0.proj", "P_bg377", "P_bg378", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.ff.net.2", "P_bg379", "P_bg380", bias=True), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.norm1", "P_bg381"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.norm2", "P_bg381"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.8.norm3", "P_bg381"), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.attn1.to_k", "P_bg382", "P_bg383", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.attn1.to_out.0", "P_bg382", "P_bg383", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.attn1.to_q", "P_bg382", "P_bg383", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.attn1.to_v", "P_bg382", "P_bg383", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.attn2.to_k", "P_bg384", "P_bg385", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.attn2.to_out.0", "P_bg386", "P_bg387", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.attn2.to_q", "P_bg388", "P_bg389", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.attn2.to_v", "P_bg390", "P_bg391", bias=False), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.ff.net.0.proj", "P_bg392", "P_bg393", bias=True), + **dense("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.ff.net.2", "P_bg394", "P_bg395", bias=True), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.norm1", "P_bg396"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.norm2", "P_bg396"), + **norm("model.diffusion_model.input_blocks.8.1.transformer_blocks.9.norm3", "P_bg396"), + **dense("model.diffusion_model.label_emb.0.0", "P_bg397", "P_bg398", bias=True), + **dense("model.diffusion_model.label_emb.0.2", "P_bg399", "P_bg400", bias=True), + **easyblock("model.diffusion_model.middle_block.0", "P_bg401", "P_bg402"), + **norm("model.diffusion_model.middle_block.1.norm", "P_bg403"), + **conv("model.diffusion_model.middle_block.1.proj_in", "P_bg403", "P_bg404"), + **conv("model.diffusion_model.middle_block.1.proj_out", "P_bg405", "P_bg406"), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_k", "P_bg407", "P_bg408", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_out.0", "P_bg407", "P_bg408", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_q", "P_bg407", "P_bg408", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_v", "P_bg407", "P_bg408", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_k", "P_bg409", "P_bg410", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_out.0", "P_bg411", "P_bg412", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_q", "P_bg413", "P_bg414", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_v", "P_bg415", "P_bg416", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.ff.net.0.proj", "P_bg417", "P_bg418", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.0.ff.net.2", "P_bg419", "P_bg420", bias=True), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.0.norm1", "P_bg421"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.0.norm2", "P_bg421"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.0.norm3", "P_bg421"), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.1.attn1.to_k", "P_bg422", "P_bg423", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.1.attn1.to_out.0", "P_bg422", "P_bg423", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.1.attn1.to_q", "P_bg422", "P_bg423", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.1.attn1.to_v", "P_bg422", "P_bg423", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.1.attn2.to_k", "P_bg424", "P_bg425", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.1.attn2.to_out.0", "P_bg426", "P_bg427", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.1.attn2.to_q", "P_bg428", "P_bg429", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.1.attn2.to_v", "P_bg430", "P_bg431", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.1.ff.net.0.proj", "P_bg432", "P_bg433", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.1.ff.net.2", "P_bg434", "P_bg435", bias=True), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.1.norm1", "P_bg436"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.1.norm2", "P_bg436"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.1.norm3", "P_bg436"), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.2.attn1.to_k", "P_bg437", "P_bg438", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.2.attn1.to_out.0", "P_bg437", "P_bg438", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.2.attn1.to_q", "P_bg437", "P_bg438", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.2.attn1.to_v", "P_bg437", "P_bg438", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.2.attn2.to_k", "P_bg439", "P_bg440", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.2.attn2.to_out.0", "P_bg441", "P_bg442", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.2.attn2.to_q", "P_bg443", "P_bg444", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.2.attn2.to_v", "P_bg445", "P_bg446", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.2.ff.net.0.proj", "P_bg447", "P_bg448", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.2.ff.net.2", "P_bg449", "P_bg450", bias=True), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.2.norm1", "P_bg451"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.2.norm2", "P_bg451"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.2.norm3", "P_bg451"), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.3.attn1.to_k", "P_bg452", "P_bg453", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.3.attn1.to_out.0", "P_bg452", "P_bg453", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.3.attn1.to_q", "P_bg452", "P_bg453", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.3.attn1.to_v", "P_bg452", "P_bg453", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.3.attn2.to_k", "P_bg454", "P_bg455", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.3.attn2.to_out.0", "P_bg456", "P_bg457", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.3.attn2.to_q", "P_bg458", "P_bg459", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.3.attn2.to_v", "P_bg460", "P_bg461", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.3.ff.net.0.proj", "P_bg462", "P_bg463", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.3.ff.net.2", "P_bg464", "P_bg465", bias=True), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.3.norm1", "P_bg466"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.3.norm2", "P_bg466"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.3.norm3", "P_bg466"), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.4.attn1.to_k", "P_bg467", "P_bg468", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.4.attn1.to_out.0", "P_bg467", "P_bg468", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.4.attn1.to_q", "P_bg467", "P_bg468", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.4.attn1.to_v", "P_bg467", "P_bg468", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.4.attn2.to_k", "P_bg469", "P_bg470", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.4.attn2.to_out.0", "P_bg471", "P_bg472", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.4.attn2.to_q", "P_bg473", "P_bg474", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.4.attn2.to_v", "P_bg475", "P_bg476", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.4.ff.net.0.proj", "P_bg477", "P_bg478", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.4.ff.net.2", "P_bg479", "P_bg480", bias=True), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.4.norm1", "P_bg481"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.4.norm2", "P_bg481"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.4.norm3", "P_bg481"), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.5.attn1.to_k", "P_bg482", "P_bg483", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.5.attn1.to_out.0", "P_bg482", "P_bg483", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.5.attn1.to_q", "P_bg482", "P_bg483", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.5.attn1.to_v", "P_bg482", "P_bg483", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.5.attn2.to_k", "P_bg484", "P_bg485", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.5.attn2.to_out.0", "P_bg486", "P_bg487", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.5.attn2.to_q", "P_bg488", "P_bg489", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.5.attn2.to_v", "P_bg490", "P_bg491", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.5.ff.net.0.proj", "P_bg492", "P_bg493", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.5.ff.net.2", "P_bg494", "P_bg495", bias=True), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.5.norm1", "P_bg496"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.5.norm2", "P_bg496"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.5.norm3", "P_bg496"), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.6.attn1.to_k", "P_bg497", "P_bg498", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.6.attn1.to_out.0", "P_bg497", "P_bg498", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.6.attn1.to_q", "P_bg497", "P_bg498", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.6.attn1.to_v", "P_bg497", "P_bg498", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.6.attn2.to_k", "P_bg499", "P_bg500", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.6.attn2.to_out.0", "P_bg501", "P_bg502", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.6.attn2.to_q", "P_bg503", "P_bg504", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.6.attn2.to_v", "P_bg505", "P_bg506", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.6.ff.net.0.proj", "P_bg507", "P_bg508", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.6.ff.net.2", "P_bg509", "P_bg510", bias=True), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.6.norm1", "P_bg511"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.6.norm2", "P_bg511"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.6.norm3", "P_bg511"), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.7.attn1.to_k", "P_bg512", "P_bg513", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.7.attn1.to_out.0", "P_bg512", "P_bg513", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.7.attn1.to_q", "P_bg512", "P_bg513", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.7.attn1.to_v", "P_bg512", "P_bg513", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.7.attn2.to_k", "P_bg514", "P_bg515", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.7.attn2.to_out.0", "P_bg516", "P_bg517", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.7.attn2.to_q", "P_bg518", "P_bg519", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.7.attn2.to_v", "P_bg520", "P_bg521", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.7.ff.net.0.proj", "P_bg522", "P_bg523", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.7.ff.net.2", "P_bg524", "P_bg525", bias=True), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.7.norm1", "P_bg526"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.7.norm2", "P_bg526"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.7.norm3", "P_bg526"), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.8.attn1.to_k", "P_bg527", "P_bg528", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.8.attn1.to_out.0", "P_bg527", "P_bg528", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.8.attn1.to_q", "P_bg527", "P_bg528", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.8.attn1.to_v", "P_bg527", "P_bg528", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.8.attn2.to_k", "P_bg529", "P_bg530", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.8.attn2.to_out.0", "P_bg531", "P_bg532", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.8.attn2.to_q", "P_bg533", "P_bg534", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.8.attn2.to_v", "P_bg535", "P_bg536", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.8.ff.net.0.proj", "P_bg537", "P_bg538", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.8.ff.net.2", "P_bg539", "P_bg540", bias=True), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.8.norm1", "P_bg541"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.8.norm2", "P_bg541"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.8.norm3", "P_bg541"), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.9.attn1.to_k", "P_bg542", "P_bg543", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.9.attn1.to_out.0", "P_bg542", "P_bg543", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.9.attn1.to_q", "P_bg542", "P_bg543", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.9.attn1.to_v", "P_bg542", "P_bg543", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.9.attn2.to_k", "P_bg544", "P_bg545", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.9.attn2.to_out.0", "P_bg546", "P_bg547", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.9.attn2.to_q", "P_bg548", "P_bg549", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.9.attn2.to_v", "P_bg550", "P_bg551", bias=False), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.9.ff.net.0.proj", "P_bg552", "P_bg553", bias=True), + **dense("model.diffusion_model.middle_block.1.transformer_blocks.9.ff.net.2", "P_bg554", "P_bg555", bias=True), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.9.norm1", "P_bg556"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.9.norm2", "P_bg556"), + **norm("model.diffusion_model.middle_block.1.transformer_blocks.9.norm3", "P_bg556"), + **easyblock("model.diffusion_model.middle_block.2", "P_bg557", "P_bg558"), + **norm("model.diffusion_model.out.0", "P_bg559"), + **conv("model.diffusion_model.out.2", "P_bg560", "P_bg561"), + **easyblock("model.diffusion_model.output_blocks.0.0", "P_bg562", "P_bg563"), + **conv("model.diffusion_model.output_blocks.0.0.skip_connection", "P_bg564", "P_bg565"), + **norm("model.diffusion_model.output_blocks.0.1.norm", "P_bg566"), + **conv("model.diffusion_model.output_blocks.0.1.proj_in", "P_bg566", "P_bg567"), + **conv("model.diffusion_model.output_blocks.0.1.proj_out", "P_bg568", "P_bg569"), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.attn1.to_k", "P_bg570", "P_bg571", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.attn1.to_out.0", "P_bg570", "P_bg571", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.attn1.to_q", "P_bg570", "P_bg571", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.attn1.to_v", "P_bg570", "P_bg571", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.attn2.to_k", "P_bg572", "P_bg573", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.attn2.to_out.0", "P_bg574", "P_bg575", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.attn2.to_q", "P_bg576", "P_bg577", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.attn2.to_v", "P_bg578", "P_bg579", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.ff.net.0.proj", "P_bg580", "P_bg581", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.ff.net.2", "P_bg582", "P_bg583", bias=True), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.norm1", "P_bg584"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.norm2", "P_bg584"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.0.norm3", "P_bg584"), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.attn1.to_k", "P_bg585", "P_bg586", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.attn1.to_out.0", "P_bg585", "P_bg586", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.attn1.to_q", "P_bg585", "P_bg586", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.attn1.to_v", "P_bg585", "P_bg586", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.attn2.to_k", "P_bg587", "P_bg588", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.attn2.to_out.0", "P_bg589", "P_bg590", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.attn2.to_q", "P_bg591", "P_bg592", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.attn2.to_v", "P_bg593", "P_bg594", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.ff.net.0.proj", "P_bg595", "P_bg596", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.ff.net.2", "P_bg597", "P_bg598", bias=True), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.norm1", "P_bg599"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.norm2", "P_bg599"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.1.norm3", "P_bg599"), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.attn1.to_k", "P_bg600", "P_bg601", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.attn1.to_out.0", "P_bg600", "P_bg601", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.attn1.to_q", "P_bg600", "P_bg601", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.attn1.to_v", "P_bg600", "P_bg601", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.attn2.to_k", "P_bg602", "P_bg603", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.attn2.to_out.0", "P_bg604", "P_bg605", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.attn2.to_q", "P_bg606", "P_bg607", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.attn2.to_v", "P_bg608", "P_bg609", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.ff.net.0.proj", "P_bg610", "P_bg611", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.ff.net.2", "P_bg612", "P_bg613", bias=True), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.norm1", "P_bg614"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.norm2", "P_bg614"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.2.norm3", "P_bg614"), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.attn1.to_k", "P_bg615", "P_bg616", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.attn1.to_out.0", "P_bg615", "P_bg616", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.attn1.to_q", "P_bg615", "P_bg616", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.attn1.to_v", "P_bg615", "P_bg616", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.attn2.to_k", "P_bg617", "P_bg618", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.attn2.to_out.0", "P_bg619", "P_bg620", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.attn2.to_q", "P_bg621", "P_bg622", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.attn2.to_v", "P_bg623", "P_bg624", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.ff.net.0.proj", "P_bg625", "P_bg626", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.ff.net.2", "P_bg627", "P_bg628", bias=True), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.norm1", "P_bg629"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.norm2", "P_bg629"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.3.norm3", "P_bg629"), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.attn1.to_k", "P_bg630", "P_bg631", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.attn1.to_out.0", "P_bg630", "P_bg631", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.attn1.to_q", "P_bg630", "P_bg631", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.attn1.to_v", "P_bg630", "P_bg631", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.attn2.to_k", "P_bg632", "P_bg633", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.attn2.to_out.0", "P_bg634", "P_bg635", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.attn2.to_q", "P_bg636", "P_bg637", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.attn2.to_v", "P_bg638", "P_bg639", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.ff.net.0.proj", "P_bg640", "P_bg641", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.ff.net.2", "P_bg642", "P_bg643", bias=True), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.norm1", "P_bg644"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.norm2", "P_bg644"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.4.norm3", "P_bg644"), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.attn1.to_k", "P_bg645", "P_bg646", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.attn1.to_out.0", "P_bg645", "P_bg646", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.attn1.to_q", "P_bg645", "P_bg646", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.attn1.to_v", "P_bg645", "P_bg646", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.attn2.to_k", "P_bg647", "P_bg648", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.attn2.to_out.0", "P_bg649", "P_bg650", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.attn2.to_q", "P_bg651", "P_bg652", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.attn2.to_v", "P_bg653", "P_bg654", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.ff.net.0.proj", "P_bg655", "P_bg656", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.ff.net.2", "P_bg657", "P_bg658", bias=True), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.norm1", "P_bg659"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.norm2", "P_bg659"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.5.norm3", "P_bg659"), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.attn1.to_k", "P_bg660", "P_bg661", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.attn1.to_out.0", "P_bg660", "P_bg661", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.attn1.to_q", "P_bg660", "P_bg661", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.attn1.to_v", "P_bg660", "P_bg661", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.attn2.to_k", "P_bg662", "P_bg663", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.attn2.to_out.0", "P_bg664", "P_bg665", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.attn2.to_q", "P_bg666", "P_bg667", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.attn2.to_v", "P_bg668", "P_bg669", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.ff.net.0.proj", "P_bg670", "P_bg671", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.ff.net.2", "P_bg672", "P_bg673", bias=True), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.norm1", "P_bg674"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.norm2", "P_bg674"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.6.norm3", "P_bg674"), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.attn1.to_k", "P_bg675", "P_bg676", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.attn1.to_out.0", "P_bg675", "P_bg676", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.attn1.to_q", "P_bg675", "P_bg676", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.attn1.to_v", "P_bg675", "P_bg676", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.attn2.to_k", "P_bg677", "P_bg678", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.attn2.to_out.0", "P_bg679", "P_bg680", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.attn2.to_q", "P_bg681", "P_bg682", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.attn2.to_v", "P_bg683", "P_bg684", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.ff.net.0.proj", "P_bg685", "P_bg686", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.ff.net.2", "P_bg687", "P_bg688", bias=True), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.norm1", "P_bg689"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.norm2", "P_bg689"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.7.norm3", "P_bg689"), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.attn1.to_k", "P_bg690", "P_bg691", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.attn1.to_out.0", "P_bg690", "P_bg691", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.attn1.to_q", "P_bg690", "P_bg691", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.attn1.to_v", "P_bg690", "P_bg691", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.attn2.to_k", "P_bg692", "P_bg693", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.attn2.to_out.0", "P_bg694", "P_bg695", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.attn2.to_q", "P_bg696", "P_bg697", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.attn2.to_v", "P_bg698", "P_bg699", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.ff.net.0.proj", "P_bg700", "P_bg701", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.ff.net.2", "P_bg702", "P_bg703", bias=True), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.norm1", "P_bg704"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.norm2", "P_bg704"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.8.norm3", "P_bg704"), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.attn1.to_k", "P_bg705", "P_bg706", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.attn1.to_out.0", "P_bg705", "P_bg706", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.attn1.to_q", "P_bg705", "P_bg706", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.attn1.to_v", "P_bg705", "P_bg706", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.attn2.to_k", "P_bg707", "P_bg708", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.attn2.to_out.0", "P_bg709", "P_bg710", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.attn2.to_q", "P_bg711", "P_bg712", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.attn2.to_v", "P_bg713", "P_bg714", bias=False), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.ff.net.0.proj", "P_bg715", "P_bg716", bias=True), + **dense("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.ff.net.2", "P_bg717", "P_bg718", bias=True), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.norm1", "P_bg719"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.norm2", "P_bg719"), + **norm("model.diffusion_model.output_blocks.0.1.transformer_blocks.9.norm3", "P_bg719"), + **easyblock("model.diffusion_model.output_blocks.1.0", "P_bg720", "P_bg721"), + **conv("model.diffusion_model.output_blocks.1.0.skip_connection", "P_bg722", "P_bg723"), + **norm("model.diffusion_model.output_blocks.1.1.norm", "P_bg724"), + **conv("model.diffusion_model.output_blocks.1.1.proj_in", "P_bg724", "P_bg725"), + **conv("model.diffusion_model.output_blocks.1.1.proj_out", "P_bg726", "P_bg727"), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.attn1.to_k", "P_bg728", "P_bg729", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.attn1.to_out.0", "P_bg728", "P_bg729", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.attn1.to_q", "P_bg728", "P_bg729", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.attn1.to_v", "P_bg728", "P_bg729", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.attn2.to_k", "P_bg730", "P_bg731", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.attn2.to_out.0", "P_bg732", "P_bg733", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.attn2.to_q", "P_bg734", "P_bg735", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.attn2.to_v", "P_bg736", "P_bg737", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.ff.net.0.proj", "P_bg738", "P_bg739", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.ff.net.2", "P_bg740", "P_bg741", bias=True), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.norm1", "P_bg742"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.norm2", "P_bg742"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.0.norm3", "P_bg742"), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.attn1.to_k", "P_bg743", "P_bg744", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.attn1.to_out.0", "P_bg743", "P_bg744", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.attn1.to_q", "P_bg743", "P_bg744", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.attn1.to_v", "P_bg743", "P_bg744", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.attn2.to_k", "P_bg745", "P_bg746", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.attn2.to_out.0", "P_bg747", "P_bg748", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.attn2.to_q", "P_bg749", "P_bg750", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.attn2.to_v", "P_bg751", "P_bg752", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.ff.net.0.proj", "P_bg753", "P_bg754", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.ff.net.2", "P_bg755", "P_bg756", bias=True), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.norm1", "P_bg757"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.norm2", "P_bg757"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.1.norm3", "P_bg757"), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.attn1.to_k", "P_bg758", "P_bg759", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.attn1.to_out.0", "P_bg758", "P_bg759", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.attn1.to_q", "P_bg758", "P_bg759", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.attn1.to_v", "P_bg758", "P_bg759", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.attn2.to_k", "P_bg760", "P_bg761", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.attn2.to_out.0", "P_bg762", "P_bg763", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.attn2.to_q", "P_bg764", "P_bg765", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.attn2.to_v", "P_bg766", "P_bg767", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.ff.net.0.proj", "P_bg768", "P_bg769", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.ff.net.2", "P_bg770", "P_bg771", bias=True), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.norm1", "P_bg772"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.norm2", "P_bg772"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.2.norm3", "P_bg772"), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.attn1.to_k", "P_bg773", "P_bg774", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.attn1.to_out.0", "P_bg773", "P_bg774", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.attn1.to_q", "P_bg773", "P_bg774", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.attn1.to_v", "P_bg773", "P_bg774", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.attn2.to_k", "P_bg775", "P_bg776", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.attn2.to_out.0", "P_bg777", "P_bg778", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.attn2.to_q", "P_bg779", "P_bg780", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.attn2.to_v", "P_bg781", "P_bg782", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.ff.net.0.proj", "P_bg783", "P_bg784", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.ff.net.2", "P_bg785", "P_bg786", bias=True), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.norm1", "P_bg787"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.norm2", "P_bg787"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.3.norm3", "P_bg787"), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.attn1.to_k", "P_bg788", "P_bg789", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.attn1.to_out.0", "P_bg788", "P_bg789", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.attn1.to_q", "P_bg788", "P_bg789", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.attn1.to_v", "P_bg788", "P_bg789", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.attn2.to_k", "P_bg790", "P_bg791", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.attn2.to_out.0", "P_bg792", "P_bg793", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.attn2.to_q", "P_bg794", "P_bg795", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.attn2.to_v", "P_bg796", "P_bg797", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.ff.net.0.proj", "P_bg798", "P_bg799", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.ff.net.2", "P_bg800", "P_bg801", bias=True), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.norm1", "P_bg802"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.norm2", "P_bg802"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.4.norm3", "P_bg802"), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.attn1.to_k", "P_bg803", "P_bg804", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.attn1.to_out.0", "P_bg803", "P_bg804", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.attn1.to_q", "P_bg803", "P_bg804", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.attn1.to_v", "P_bg803", "P_bg804", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.attn2.to_k", "P_bg805", "P_bg806", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.attn2.to_out.0", "P_bg807", "P_bg808", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.attn2.to_q", "P_bg809", "P_bg810", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.attn2.to_v", "P_bg811", "P_bg812", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.ff.net.0.proj", "P_bg813", "P_bg814", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.ff.net.2", "P_bg815", "P_bg816", bias=True), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.norm1", "P_bg817"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.norm2", "P_bg817"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.5.norm3", "P_bg817"), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.attn1.to_k", "P_bg818", "P_bg819", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.attn1.to_out.0", "P_bg818", "P_bg819", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.attn1.to_q", "P_bg818", "P_bg819", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.attn1.to_v", "P_bg818", "P_bg819", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.attn2.to_k", "P_bg820", "P_bg821", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.attn2.to_out.0", "P_bg822", "P_bg823", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.attn2.to_q", "P_bg824", "P_bg825", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.attn2.to_v", "P_bg826", "P_bg827", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.ff.net.0.proj", "P_bg828", "P_bg829", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.ff.net.2", "P_bg830", "P_bg831", bias=True), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.norm1", "P_bg832"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.norm2", "P_bg832"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.6.norm3", "P_bg832"), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.attn1.to_k", "P_bg833", "P_bg834", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.attn1.to_out.0", "P_bg833", "P_bg834", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.attn1.to_q", "P_bg833", "P_bg834", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.attn1.to_v", "P_bg833", "P_bg834", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.attn2.to_k", "P_bg835", "P_bg836", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.attn2.to_out.0", "P_bg837", "P_bg838", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.attn2.to_q", "P_bg839", "P_bg840", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.attn2.to_v", "P_bg841", "P_bg842", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.ff.net.0.proj", "P_bg843", "P_bg844", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.ff.net.2", "P_bg845", "P_bg846", bias=True), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.norm1", "P_bg847"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.norm2", "P_bg847"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.7.norm3", "P_bg847"), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.attn1.to_k", "P_bg848", "P_bg849", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.attn1.to_out.0", "P_bg848", "P_bg849", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.attn1.to_q", "P_bg848", "P_bg849", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.attn1.to_v", "P_bg848", "P_bg849", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.attn2.to_k", "P_bg850", "P_bg851", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.attn2.to_out.0", "P_bg852", "P_bg853", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.attn2.to_q", "P_bg854", "P_bg855", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.attn2.to_v", "P_bg856", "P_bg857", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.ff.net.0.proj", "P_bg858", "P_bg859", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.ff.net.2", "P_bg860", "P_bg861", bias=True), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.norm1", "P_bg862"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.norm2", "P_bg862"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.8.norm3", "P_bg862"), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.attn1.to_k", "P_bg863", "P_bg864", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.attn1.to_out.0", "P_bg863", "P_bg864", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.attn1.to_q", "P_bg863", "P_bg864", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.attn1.to_v", "P_bg863", "P_bg864", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.attn2.to_k", "P_bg865", "P_bg866", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.attn2.to_out.0", "P_bg867", "P_bg868", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.attn2.to_q", "P_bg869", "P_bg870", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.attn2.to_v", "P_bg871", "P_bg872", bias=False), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.ff.net.0.proj", "P_bg873", "P_bg874", bias=True), + **dense("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.ff.net.2", "P_bg875", "P_bg876", bias=True), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.norm1", "P_bg877"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.norm2", "P_bg877"), + **norm("model.diffusion_model.output_blocks.1.1.transformer_blocks.9.norm3", "P_bg877"), + **easyblock("model.diffusion_model.output_blocks.2.0", "P_bg878", "P_bg879"), + **conv("model.diffusion_model.output_blocks.2.0.skip_connection", "P_bg880", "P_bg881"), + **norm("model.diffusion_model.output_blocks.2.1.norm", "P_bg882"), + **conv("model.diffusion_model.output_blocks.2.1.proj_in", "P_bg882", "P_bg883"), + **conv("model.diffusion_model.output_blocks.2.1.proj_out", "P_bg884", "P_bg885"), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.attn1.to_k", "P_bg886", "P_bg887", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.attn1.to_out.0", "P_bg886", "P_bg887", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.attn1.to_q", "P_bg886", "P_bg887", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.attn1.to_v", "P_bg886", "P_bg887", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.attn2.to_k", "P_bg888", "P_bg889", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.attn2.to_out.0", "P_bg890", "P_bg891", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.attn2.to_q", "P_bg892", "P_bg893", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.attn2.to_v", "P_bg894", "P_bg895", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.ff.net.0.proj", "P_bg896", "P_bg897", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.ff.net.2", "P_bg898", "P_bg899", bias=True), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.norm1", "P_bg900"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.norm2", "P_bg900"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.0.norm3", "P_bg900"), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.attn1.to_k", "P_bg901", "P_bg902", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.attn1.to_out.0", "P_bg901", "P_bg902", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.attn1.to_q", "P_bg901", "P_bg902", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.attn1.to_v", "P_bg901", "P_bg902", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.attn2.to_k", "P_bg903", "P_bg904", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.attn2.to_out.0", "P_bg905", "P_bg906", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.attn2.to_q", "P_bg907", "P_bg908", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.attn2.to_v", "P_bg909", "P_bg910", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.ff.net.0.proj", "P_bg911", "P_bg912", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.ff.net.2", "P_bg913", "P_bg914", bias=True), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.norm1", "P_bg915"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.norm2", "P_bg915"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.1.norm3", "P_bg915"), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.attn1.to_k", "P_bg916", "P_bg917", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.attn1.to_out.0", "P_bg916", "P_bg917", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.attn1.to_q", "P_bg916", "P_bg917", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.attn1.to_v", "P_bg916", "P_bg917", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.attn2.to_k", "P_bg918", "P_bg919", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.attn2.to_out.0", "P_bg920", "P_bg921", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.attn2.to_q", "P_bg922", "P_bg923", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.attn2.to_v", "P_bg924", "P_bg925", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.ff.net.0.proj", "P_bg926", "P_bg927", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.ff.net.2", "P_bg928", "P_bg929", bias=True), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.norm1", "P_bg930"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.norm2", "P_bg930"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.2.norm3", "P_bg930"), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.attn1.to_k", "P_bg931", "P_bg932", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.attn1.to_out.0", "P_bg931", "P_bg932", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.attn1.to_q", "P_bg931", "P_bg932", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.attn1.to_v", "P_bg931", "P_bg932", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.attn2.to_k", "P_bg933", "P_bg934", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.attn2.to_out.0", "P_bg935", "P_bg936", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.attn2.to_q", "P_bg937", "P_bg938", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.attn2.to_v", "P_bg939", "P_bg940", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.ff.net.0.proj", "P_bg941", "P_bg942", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.ff.net.2", "P_bg943", "P_bg944", bias=True), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.norm1", "P_bg945"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.norm2", "P_bg945"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.3.norm3", "P_bg945"), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.attn1.to_k", "P_bg946", "P_bg947", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.attn1.to_out.0", "P_bg946", "P_bg947", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.attn1.to_q", "P_bg946", "P_bg947", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.attn1.to_v", "P_bg946", "P_bg947", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.attn2.to_k", "P_bg948", "P_bg949", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.attn2.to_out.0", "P_bg950", "P_bg951", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.attn2.to_q", "P_bg952", "P_bg953", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.attn2.to_v", "P_bg954", "P_bg955", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.ff.net.0.proj", "P_bg956", "P_bg957", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.ff.net.2", "P_bg958", "P_bg959", bias=True), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.norm1", "P_bg960"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.norm2", "P_bg960"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.4.norm3", "P_bg960"), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.attn1.to_k", "P_bg961", "P_bg962", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.attn1.to_out.0", "P_bg961", "P_bg962", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.attn1.to_q", "P_bg961", "P_bg962", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.attn1.to_v", "P_bg961", "P_bg962", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.attn2.to_k", "P_bg963", "P_bg964", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.attn2.to_out.0", "P_bg965", "P_bg966", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.attn2.to_q", "P_bg967", "P_bg968", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.attn2.to_v", "P_bg969", "P_bg970", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.ff.net.0.proj", "P_bg971", "P_bg972", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.ff.net.2", "P_bg973", "P_bg974", bias=True), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.norm1", "P_bg975"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.norm2", "P_bg975"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.5.norm3", "P_bg975"), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.attn1.to_k", "P_bg976", "P_bg977", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.attn1.to_out.0", "P_bg976", "P_bg977", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.attn1.to_q", "P_bg976", "P_bg977", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.attn1.to_v", "P_bg976", "P_bg977", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.attn2.to_k", "P_bg978", "P_bg979", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.attn2.to_out.0", "P_bg980", "P_bg981", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.attn2.to_q", "P_bg982", "P_bg983", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.attn2.to_v", "P_bg984", "P_bg985", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.ff.net.0.proj", "P_bg986", "P_bg987", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.ff.net.2", "P_bg988", "P_bg989", bias=True), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.norm1", "P_bg990"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.norm2", "P_bg990"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.6.norm3", "P_bg990"), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.attn1.to_k", "P_bg991", "P_bg992", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.attn1.to_out.0", "P_bg991", "P_bg992", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.attn1.to_q", "P_bg991", "P_bg992", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.attn1.to_v", "P_bg991", "P_bg992", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.attn2.to_k", "P_bg993", "P_bg994", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.attn2.to_out.0", "P_bg995", "P_bg996", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.attn2.to_q", "P_bg997", "P_bg998", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.attn2.to_v", "P_bg999", "P_bg1000", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.ff.net.0.proj", "P_bg1001", "P_bg1002", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.ff.net.2", "P_bg1003", "P_bg1004", bias=True), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.norm1", "P_bg1005"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.norm2", "P_bg1005"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.7.norm3", "P_bg1005"), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.attn1.to_k", "P_bg1006", "P_bg1007", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.attn1.to_out.0", "P_bg1006", "P_bg1007", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.attn1.to_q", "P_bg1006", "P_bg1007", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.attn1.to_v", "P_bg1006", "P_bg1007", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.attn2.to_k", "P_bg1008", "P_bg1009", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.attn2.to_out.0", "P_bg1010", "P_bg1011", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.attn2.to_q", "P_bg1012", "P_bg1013", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.attn2.to_v", "P_bg1014", "P_bg1015", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.ff.net.0.proj", "P_bg1016", "P_bg1017", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.ff.net.2", "P_bg1018", "P_bg1019", bias=True), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.norm1", "P_bg1020"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.norm2", "P_bg1020"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.8.norm3", "P_bg1020"), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.attn1.to_k", "P_bg1021", "P_bg1022", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.attn1.to_out.0", "P_bg1021", "P_bg1022", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.attn1.to_q", "P_bg1021", "P_bg1022", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.attn1.to_v", "P_bg1021", "P_bg1022", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.attn2.to_k", "P_bg1023", "P_bg1024", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.attn2.to_out.0", "P_bg1025", "P_bg1026", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.attn2.to_q", "P_bg1027", "P_bg1028", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.attn2.to_v", "P_bg1029", "P_bg1030", bias=False), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.ff.net.0.proj", "P_bg1031", "P_bg1032", bias=True), + **dense("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.ff.net.2", "P_bg1033", "P_bg1034", bias=True), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.norm1", "P_bg1035"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.norm2", "P_bg1035"), + **norm("model.diffusion_model.output_blocks.2.1.transformer_blocks.9.norm3", "P_bg1035"), + **conv("model.diffusion_model.output_blocks.2.2.conv", "P_bg1036", "P_bg1037"), + **easyblock("model.diffusion_model.output_blocks.3.0", "P_bg1038", "P_bg1039"), + **conv("model.diffusion_model.output_blocks.3.0.skip_connection", "P_bg1040", "P_bg1041"), + **norm("model.diffusion_model.output_blocks.3.1.norm", "P_bg1042"), + **conv("model.diffusion_model.output_blocks.3.1.proj_in", "P_bg1042", "P_bg1043"), + **conv("model.diffusion_model.output_blocks.3.1.proj_out", "P_bg1044", "P_bg1045"), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_k", "P_bg1046", "P_bg1047", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_out.0", "P_bg1046", "P_bg1047", bias=True), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_q", "P_bg1046", "P_bg1047", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_v", "P_bg1046", "P_bg1047", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_k", "P_bg1048", "P_bg1049", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_out.0", "P_bg1050", "P_bg1051", bias=True), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_q", "P_bg1052", "P_bg1053", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_v", "P_bg1054", "P_bg1055", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.ff.net.0.proj", "P_bg1056", "P_bg1057", bias=True), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.ff.net.2", "P_bg1058", "P_bg1059", bias=True), + **norm("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.norm1", "P_bg1060"), + **norm("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.norm2", "P_bg1060"), + **norm("model.diffusion_model.output_blocks.3.1.transformer_blocks.0.norm3", "P_bg1060"), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.attn1.to_k", "P_bg1061", "P_bg1062", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.attn1.to_out.0", "P_bg1061", "P_bg1062", bias=True), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.attn1.to_q", "P_bg1061", "P_bg1062", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.attn1.to_v", "P_bg1061", "P_bg1062", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.attn2.to_k", "P_bg1063", "P_bg1064", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.attn2.to_out.0", "P_bg1065", "P_bg1066", bias=True), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.attn2.to_q", "P_bg1067", "P_bg1068", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.attn2.to_v", "P_bg1069", "P_bg1070", bias=False), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.ff.net.0.proj", "P_bg1071", "P_bg1072", bias=True), + **dense("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.ff.net.2", "P_bg1073", "P_bg1074", bias=True), + **norm("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.norm1", "P_bg1075"), + **norm("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.norm2", "P_bg1075"), + **norm("model.diffusion_model.output_blocks.3.1.transformer_blocks.1.norm3", "P_bg1075"), + **easyblock("model.diffusion_model.output_blocks.4.0", "P_bg1076", "P_bg1077"), + **conv("model.diffusion_model.output_blocks.4.0.skip_connection", "P_bg1078", "P_bg1079"), + **norm("model.diffusion_model.output_blocks.4.1.norm", "P_bg1080"), + **conv("model.diffusion_model.output_blocks.4.1.proj_in", "P_bg1080", "P_bg1081"), + **conv("model.diffusion_model.output_blocks.4.1.proj_out", "P_bg1082", "P_bg1083"), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_k", "P_bg1084", "P_bg1085", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_out.0", "P_bg1084", "P_bg1085", bias=True), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_q", "P_bg1084", "P_bg1085", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_v", "P_bg1084", "P_bg1085", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_k", "P_bg1086", "P_bg1087", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_out.0", "P_bg1088", "P_bg1089", bias=True), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_q", "P_bg1090", "P_bg1091", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_v", "P_bg1092", "P_bg1093", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.ff.net.0.proj", "P_bg1094", "P_bg1095", bias=True), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.ff.net.2", "P_bg1096", "P_bg1097", bias=True), + **norm("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.norm1", "P_bg1098"), + **norm("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.norm2", "P_bg1098"), + **norm("model.diffusion_model.output_blocks.4.1.transformer_blocks.0.norm3", "P_bg1098"), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.attn1.to_k", "P_bg1099", "P_bg1100", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.attn1.to_out.0", "P_bg1099", "P_bg1100", bias=True), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.attn1.to_q", "P_bg1099", "P_bg1100", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.attn1.to_v", "P_bg1099", "P_bg1100", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.attn2.to_k", "P_bg1101", "P_bg1102", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.attn2.to_out.0", "P_bg1103", "P_bg1104", bias=True), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.attn2.to_q", "P_bg1105", "P_bg1106", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.attn2.to_v", "P_bg1107", "P_bg1108", bias=False), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.ff.net.0.proj", "P_bg1109", "P_bg1110", bias=True), + **dense("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.ff.net.2", "P_bg1111", "P_bg1112", bias=True), + **norm("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.norm1", "P_bg1113"), + **norm("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.norm2", "P_bg1113"), + **norm("model.diffusion_model.output_blocks.4.1.transformer_blocks.1.norm3", "P_bg1113"), + **easyblock("model.diffusion_model.output_blocks.5.0", "P_bg1114", "P_bg1115"), + **conv("model.diffusion_model.output_blocks.5.0.skip_connection", "P_bg1116", "P_bg1117"), + **norm("model.diffusion_model.output_blocks.5.1.norm", "P_bg1118"), + **conv("model.diffusion_model.output_blocks.5.1.proj_in", "P_bg1118", "P_bg1119"), + **conv("model.diffusion_model.output_blocks.5.1.proj_out", "P_bg1120", "P_bg1121"), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_k", "P_bg1122", "P_bg1123", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_out.0", "P_bg1122", "P_bg1123", bias=True), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_q", "P_bg1122", "P_bg1123", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_v", "P_bg1122", "P_bg1123", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_k", "P_bg1124", "P_bg1125", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_out.0", "P_bg1126", "P_bg1127", bias=True), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_q", "P_bg1128", "P_bg1129", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_v", "P_bg1130", "P_bg1131", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.ff.net.0.proj", "P_bg1132", "P_bg1133", bias=True), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.ff.net.2", "P_bg1134", "P_bg1135", bias=True), + **norm("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.norm1", "P_bg1136"), + **norm("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.norm2", "P_bg1136"), + **norm("model.diffusion_model.output_blocks.5.1.transformer_blocks.0.norm3", "P_bg1136"), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.attn1.to_k", "P_bg1137", "P_bg1138", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.attn1.to_out.0", "P_bg1137", "P_bg1138", bias=True), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.attn1.to_q", "P_bg1137", "P_bg1138", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.attn1.to_v", "P_bg1137", "P_bg1138", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.attn2.to_k", "P_bg1139", "P_bg1140", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.attn2.to_out.0", "P_bg1141", "P_bg1142", bias=True), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.attn2.to_q", "P_bg1143", "P_bg1144", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.attn2.to_v", "P_bg1145", "P_bg1146", bias=False), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.ff.net.0.proj", "P_bg1147", "P_bg1148", bias=True), + **dense("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.ff.net.2", "P_bg1149", "P_bg1150", bias=True), + **norm("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.norm1", "P_bg1151"), + **norm("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.norm2", "P_bg1151"), + **norm("model.diffusion_model.output_blocks.5.1.transformer_blocks.1.norm3", "P_bg1151"), + **conv("model.diffusion_model.output_blocks.5.2.conv", "P_bg1152", "P_bg1153"), + **easyblock("model.diffusion_model.output_blocks.6.0", "P_bg1154", "P_bg1155"), + **conv("model.diffusion_model.output_blocks.6.0.skip_connection", "P_bg1156", "P_bg1157"), + **easyblock("model.diffusion_model.output_blocks.7.0", "P_bg1158", "P_bg1159"), + **conv("model.diffusion_model.output_blocks.7.0.skip_connection", "P_bg1160", "P_bg1161"), + **easyblock("model.diffusion_model.output_blocks.8.0", "P_bg1162", "P_bg1163"), + **conv("model.diffusion_model.output_blocks.8.0.skip_connection", "P_bg1164", "P_bg1165"), + **dense("model.diffusion_model.time_embed.0", "P_bg1166", "P_bg1167", bias=True), + **dense("model.diffusion_model.time_embed.2", "P_bg1168", "P_bg1169", bias=True), + # Text Encoder 1 + **dense("conditioner.embedders.0.transformer.text_model.embeddings.position_embedding", "P_bg1170", "P_bg1171", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.embeddings.token_embedding", "P_bg1172", "P_bg1173", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.mlp.fc1", "P_bg1176", "P_bg1177", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.mlp.fc2", "P_bg1178", "P_bg1179", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.k_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.out_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.q_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.v_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.mlp.fc1", "P_bg1184", "P_bg1185", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.mlp.fc2", "P_bg1186", "P_bg1187", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.k_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.out_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.q_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.v_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.mlp.fc1", "P_bg1192", "P_bg1193", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.mlp.fc2", "P_bg1194", "P_bg1195", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.k_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.out_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.q_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.v_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.mlp.fc1", "P_bg1200", "P_bg1201", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.mlp.fc2", "P_bg1202", "P_bg1203", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.k_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.out_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.q_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.v_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.mlp.fc1", "P_bg1208", "P_bg1209", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.mlp.fc2", "P_bg1210", "P_bg1211", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.k_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.out_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.q_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.v_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.mlp.fc1", "P_bg1216", "P_bg1217", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.mlp.fc2", "P_bg1218", "P_bg1219", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.k_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.out_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.q_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.v_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.mlp.fc1", "P_bg1224", "P_bg1225", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.mlp.fc2", "P_bg1226", "P_bg1227", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.k_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.out_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.q_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.v_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.mlp.fc1", "P_bg1232", "P_bg1233", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.mlp.fc2", "P_bg1234", "P_bg1235", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.k_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.out_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.q_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.v_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.mlp.fc1", "P_bg1240", "P_bg1241", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.mlp.fc2", "P_bg1242", "P_bg1243", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.k_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.out_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.q_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.v_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.mlp.fc1", "P_bg1248", "P_bg1249", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.mlp.fc2", "P_bg1250", "P_bg1251", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.k_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.out_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.q_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.v_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.mlp.fc1", "P_bg1256", "P_bg1257", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.mlp.fc2", "P_bg1258", "P_bg1259", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.k_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.out_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.q_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.v_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.mlp.fc1", "P_bg1264", "P_bg1265", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.mlp.fc2", "P_bg1266", "P_bg1267", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.k_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.out_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.q_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.v_proj", "P_bg1268", "P_bg1269", bias=False), + # Text Encoder 2 + **dense("conditioner.embedders.1.model.token_embedding", "P_bg1272", "P_bg1273", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.attn.in_proj_weight", "P_bg1274", "P_bg1275", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.attn.out_proj", "P_bg1274", "P_bg1275", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.mlp.c_fc", "P_bg1278", "P_bg1279", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.mlp.c_proj", "P_bg1280", "P_bg1281", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.attn.in_proj_weight", "P_bg1280", "P_bg1281", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.attn.out_proj", "P_bg1280", "P_bg1281", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.mlp.c_fc", "P_bg1284", "P_bg1285", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.mlp.c_proj", "P_bg1286", "P_bg1287", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.attn.in_proj_weight", "P_bg1286", "P_bg1287", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.attn.out_proj", "P_bg1286", "P_bg1287", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.mlp.c_fc", "P_bg1290", "P_bg1291", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.mlp.c_proj", "P_bg1292", "P_bg1293", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.attn.in_proj_weight", "P_bg1292", "P_bg1293", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.attn.out_proj", "P_bg1292", "P_bg1293", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.mlp.c_fc", "P_bg1296", "P_bg1297", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.mlp.c_proj", "P_bg1298", "P_bg1299", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.attn.in_proj_weight", "P_bg1298", "P_bg1299", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.attn.out_proj", "P_bg1298", "P_bg1299", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.mlp.c_fc", "P_bg1302", "P_bg1303", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.mlp.c_proj", "P_bg1304", "P_bg1305", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.attn.in_proj_weight", "P_bg1304", "P_bg1305", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.attn.out_proj", "P_bg1304", "P_bg1305", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.mlp.c_fc", "P_bg1308", "P_bg1309", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.mlp.c_proj", "P_bg1310", "P_bg1311", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.attn.in_proj_weight", "P_bg1310", "P_bg1311", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.attn.out_proj", "P_bg1310", "P_bg1311", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.mlp.c_fc", "P_bg1314", "P_bg1315", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.mlp.c_proj", "P_bg1316", "P_bg1317", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.attn.in_proj_weight", "P_bg1316", "P_bg1317", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.attn.out_proj", "P_bg1316", "P_bg1317", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.mlp.c_fc", "P_bg1320", "P_bg1321", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.mlp.c_proj", "P_bg1322", "P_bg1323", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.attn.in_proj_weight", "P_bg1322", "P_bg1323", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.attn.out_proj", "P_bg1322", "P_bg1323", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.mlp.c_fc", "P_bg1326", "P_bg1327", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.mlp.c_proj", "P_bg1328", "P_bg1329", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.attn.in_proj_weight", "P_bg1328", "P_bg1329", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.attn.out_proj", "P_bg1328", "P_bg1329", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.mlp.c_fc", "P_bg1332", "P_bg1333", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.mlp.c_proj", "P_bg1334", "P_bg1335", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.attn.in_proj_weight", "P_bg1334", "P_bg1335", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.attn.out_proj", "P_bg1334", "P_bg1335", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.mlp.c_fc", "P_bg1338", "P_bg1339", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.mlp.c_proj", "P_bg1340", "P_bg1341", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.attn.in_proj_weight", "P_bg1340", "P_bg1341", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.attn.out_proj", "P_bg1340", "P_bg1341", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.mlp.c_fc", "P_bg1344", "P_bg1345", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.mlp.c_proj", "P_bg1346", "P_bg1347", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.attn.in_proj_weight", "P_bg1346", "P_bg1347", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.attn.out_proj", "P_bg1346", "P_bg1347", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.mlp.c_fc", "P_bg1350", "P_bg1351", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.mlp.c_proj", "P_bg1352", "P_bg1353", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.attn.in_proj_weight", "P_bg1352", "P_bg1353", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.attn.out_proj", "P_bg1352", "P_bg1353", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.mlp.c_fc", "P_bg1356", "P_bg1357", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.mlp.c_proj", "P_bg1358", "P_bg1359", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.attn.in_proj_weight", "P_bg1358", "P_bg1359", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.attn.out_proj", "P_bg1358", "P_bg1359", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.mlp.c_fc", "P_bg1362", "P_bg1363", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.mlp.c_proj", "P_bg1364", "P_bg1365", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.attn.in_proj_weight", "P_bg1364", "P_bg1365", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.attn.out_proj", "P_bg1364", "P_bg1365", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.mlp.c_fc", "P_bg1368", "P_bg1369", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.mlp.c_proj", "P_bg1370", "P_bg1371", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.attn.in_proj_weight", "P_bg1370", "P_bg1371", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.attn.out_proj", "P_bg1370", "P_bg1371", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.mlp.c_fc", "P_bg1374", "P_bg1375", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.mlp.c_proj", "P_bg1376", "P_bg1377", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.attn.in_proj_weight", "P_bg1376", "P_bg1377", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.attn.out_proj", "P_bg1376", "P_bg1377", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.mlp.c_fc", "P_bg1380", "P_bg1381", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.mlp.c_proj", "P_bg1382", "P_bg1383", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.attn.in_proj_weight", "P_bg1382", "P_bg1383", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.attn.out_proj", "P_bg1382", "P_bg1383", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.mlp.c_fc", "P_bg1386", "P_bg1387", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.mlp.c_proj", "P_bg1388", "P_bg1389", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.attn.in_proj_weight", "P_bg1388", "P_bg1389", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.attn.out_proj", "P_bg1388", "P_bg1389", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.mlp.c_fc", "P_bg1392", "P_bg1393", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.mlp.c_proj", "P_bg1394", "P_bg1395", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.attn.in_proj_weight", "P_bg1394", "P_bg1395", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.attn.out_proj", "P_bg1394", "P_bg1395", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.mlp.c_fc", "P_bg1398", "P_bg1399", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.mlp.c_proj", "P_bg1400", "P_bg1401", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.attn.in_proj_weight", "P_bg1400", "P_bg1401", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.attn.out_proj", "P_bg1400", "P_bg1401", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.mlp.c_fc", "P_bg1404", "P_bg1405", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.mlp.c_proj", "P_bg1406", "P_bg1407", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.attn.in_proj_weight", "P_bg1406", "P_bg1407", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.attn.out_proj", "P_bg1406", "P_bg1407", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.mlp.c_fc", "P_bg1410", "P_bg1411", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.mlp.c_proj", "P_bg1412", "P_bg1413", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.attn.in_proj_weight", "P_bg1412", "P_bg1413", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.attn.out_proj", "P_bg1412", "P_bg1413", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.mlp.c_fc", "P_bg1416", "P_bg1417", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.mlp.c_proj", "P_bg1418", "P_bg1419", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.attn.in_proj_weight", "P_bg1418", "P_bg1419", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.attn.out_proj", "P_bg1418", "P_bg1419", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.mlp.c_fc", "P_bg1422", "P_bg1423", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.mlp.c_proj", "P_bg1424", "P_bg1425", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.attn.in_proj_weight", "P_bg1424", "P_bg1425", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.attn.out_proj", "P_bg1424", "P_bg1425", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.mlp.c_fc", "P_bg1428", "P_bg1429", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.mlp.c_proj", "P_bg1430", "P_bg1431", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.attn.in_proj_weight", "P_bg1430", "P_bg1431", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.attn.out_proj", "P_bg1430", "P_bg1431", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.mlp.c_fc", "P_bg1434", "P_bg1435", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.mlp.c_proj", "P_bg1436", "P_bg1437", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.attn.in_proj_weight", "P_bg1436", "P_bg1437", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.attn.out_proj", "P_bg1436", "P_bg1437", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.mlp.c_fc", "P_bg1440", "P_bg1441", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.mlp.c_proj", "P_bg1442", "P_bg1443", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.attn.in_proj_weight", "P_bg1442", "P_bg1443", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.attn.out_proj", "P_bg1442", "P_bg1443", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.mlp.c_fc", "P_bg1446", "P_bg1447", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.mlp.c_proj", "P_bg1448", "P_bg1449", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.attn.in_proj_weight", "P_bg1448", "P_bg1449", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.attn.out_proj", "P_bg1448", "P_bg1449", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.mlp.c_fc", "P_bg1452", "P_bg1453", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.mlp.c_proj", "P_bg1454", "P_bg1455", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.attn.in_proj_weight", "P_bg1454", "P_bg1455", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.attn.out_proj", "P_bg1454", "P_bg1455", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.mlp.c_fc", "P_bg1458", "P_bg1459", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.mlp.c_proj", "P_bg1460", "P_bg1461", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.attn.in_proj_weight", "P_bg1460", "P_bg1461", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.attn.out_proj", "P_bg1460", "P_bg1461", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.mlp.c_fc", "P_bg1464", "P_bg1465", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.mlp.c_proj", "P_bg1466", "P_bg1467", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.embeddings.position_embedding", "P_bg1170", "P_bg1171", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.embeddings.token_embedding", "P_bg1172", "P_bg1173", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.mlp.fc1", "P_bg1176", "P_bg1177", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.mlp.fc2", "P_bg1178", "P_bg1179", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.k_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.out_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.q_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.v_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.mlp.fc1", "P_bg1184", "P_bg1185", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.mlp.fc2", "P_bg1186", "P_bg1187", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.k_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.out_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.q_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.v_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.mlp.fc1", "P_bg1192", "P_bg1193", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.mlp.fc2", "P_bg1194", "P_bg1195", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.k_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.out_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.q_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.v_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.mlp.fc1", "P_bg1200", "P_bg1201", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.mlp.fc2", "P_bg1202", "P_bg1203", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.k_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.out_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.q_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.v_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.mlp.fc1", "P_bg1208", "P_bg1209", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.mlp.fc2", "P_bg1210", "P_bg1211", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.k_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.out_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.q_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.v_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.mlp.fc1", "P_bg1216", "P_bg1217", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.mlp.fc2", "P_bg1218", "P_bg1219", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.k_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.out_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.q_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.v_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.mlp.fc1", "P_bg1224", "P_bg1225", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.mlp.fc2", "P_bg1226", "P_bg1227", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.k_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.out_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.q_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.v_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.mlp.fc1", "P_bg1232", "P_bg1233", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.mlp.fc2", "P_bg1234", "P_bg1235", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.k_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.out_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.q_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.v_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.mlp.fc1", "P_bg1240", "P_bg1241", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.mlp.fc2", "P_bg1242", "P_bg1243", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.k_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.out_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.q_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.v_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.mlp.fc1", "P_bg1248", "P_bg1249", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.mlp.fc2", "P_bg1250", "P_bg1251", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.k_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.out_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.q_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.v_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.mlp.fc1", "P_bg1256", "P_bg1257", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.mlp.fc2", "P_bg1258", "P_bg1259", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.k_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.out_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.q_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.v_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.mlp.fc1", "P_bg1264", "P_bg1265", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.mlp.fc2", "P_bg1266", "P_bg1267", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.k_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.out_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.q_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.v_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.1.model.token_embedding", "P_bg1272", "P_bg1273", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.attn.in_proj_weight", "P_bg1274", "P_bg1275", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.attn.out_proj", "P_bg1274", "P_bg1275", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.mlp.c_fc", "P_bg1278", "P_bg1279", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.mlp.c_proj", "P_bg1280", "P_bg1281", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.attn.in_proj_weight", "P_bg1280", "P_bg1281", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.attn.out_proj", "P_bg1280", "P_bg1281", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.mlp.c_fc", "P_bg1284", "P_bg1285", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.mlp.c_proj", "P_bg1286", "P_bg1287", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.attn.in_proj_weight", "P_bg1286", "P_bg1287", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.attn.out_proj", "P_bg1286", "P_bg1287", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.mlp.c_fc", "P_bg1290", "P_bg1291", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.mlp.c_proj", "P_bg1292", "P_bg1293", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.attn.in_proj_weight", "P_bg1292", "P_bg1293", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.attn.out_proj", "P_bg1292", "P_bg1293", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.mlp.c_fc", "P_bg1296", "P_bg1297", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.mlp.c_proj", "P_bg1298", "P_bg1299", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.attn.in_proj_weight", "P_bg1298", "P_bg1299", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.attn.out_proj", "P_bg1298", "P_bg1299", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.mlp.c_fc", "P_bg1302", "P_bg1303", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.mlp.c_proj", "P_bg1304", "P_bg1305", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.attn.in_proj_weight", "P_bg1304", "P_bg1305", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.attn.out_proj", "P_bg1304", "P_bg1305", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.mlp.c_fc", "P_bg1308", "P_bg1309", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.mlp.c_proj", "P_bg1310", "P_bg1311", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.attn.in_proj_weight", "P_bg1310", "P_bg1311", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.attn.out_proj", "P_bg1310", "P_bg1311", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.mlp.c_fc", "P_bg1314", "P_bg1315", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.mlp.c_proj", "P_bg1316", "P_bg1317", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.attn.in_proj_weight", "P_bg1316", "P_bg1317", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.attn.out_proj", "P_bg1316", "P_bg1317", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.mlp.c_fc", "P_bg1320", "P_bg1321", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.mlp.c_proj", "P_bg1322", "P_bg1323", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.attn.in_proj_weight", "P_bg1322", "P_bg1323", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.attn.out_proj", "P_bg1322", "P_bg1323", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.mlp.c_fc", "P_bg1326", "P_bg1327", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.mlp.c_proj", "P_bg1328", "P_bg1329", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.attn.in_proj_weight", "P_bg1328", "P_bg1329", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.attn.out_proj", "P_bg1328", "P_bg1329", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.mlp.c_fc", "P_bg1332", "P_bg1333", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.mlp.c_proj", "P_bg1334", "P_bg1335", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.attn.in_proj_weight", "P_bg1334", "P_bg1335", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.attn.out_proj", "P_bg1334", "P_bg1335", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.mlp.c_fc", "P_bg1338", "P_bg1339", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.mlp.c_proj", "P_bg1340", "P_bg1341", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.attn.in_proj_weight", "P_bg1340", "P_bg1341", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.attn.out_proj", "P_bg1340", "P_bg1341", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.mlp.c_fc", "P_bg1344", "P_bg1345", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.mlp.c_proj", "P_bg1346", "P_bg1347", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.attn.in_proj_weight", "P_bg1346", "P_bg1347", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.attn.out_proj", "P_bg1346", "P_bg1347", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.mlp.c_fc", "P_bg1350", "P_bg1351", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.mlp.c_proj", "P_bg1352", "P_bg1353", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.attn.in_proj_weight", "P_bg1352", "P_bg1353", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.attn.out_proj", "P_bg1352", "P_bg1353", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.mlp.c_fc", "P_bg1356", "P_bg1357", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.mlp.c_proj", "P_bg1358", "P_bg1359", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.attn.in_proj_weight", "P_bg1358", "P_bg1359", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.attn.out_proj", "P_bg1358", "P_bg1359", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.mlp.c_fc", "P_bg1362", "P_bg1363", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.mlp.c_proj", "P_bg1364", "P_bg1365", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.attn.in_proj_weight", "P_bg1364", "P_bg1365", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.attn.out_proj", "P_bg1364", "P_bg1365", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.mlp.c_fc", "P_bg1368", "P_bg1369", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.mlp.c_proj", "P_bg1370", "P_bg1371", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.attn.in_proj_weight", "P_bg1370", "P_bg1371", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.attn.out_proj", "P_bg1370", "P_bg1371", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.mlp.c_fc", "P_bg1374", "P_bg1375", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.mlp.c_proj", "P_bg1376", "P_bg1377", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.attn.in_proj_weight", "P_bg1376", "P_bg1377", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.attn.out_proj", "P_bg1376", "P_bg1377", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.mlp.c_fc", "P_bg1380", "P_bg1381", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.mlp.c_proj", "P_bg1382", "P_bg1383", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.attn.in_proj_weight", "P_bg1382", "P_bg1383", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.attn.out_proj", "P_bg1382", "P_bg1383", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.mlp.c_fc", "P_bg1386", "P_bg1387", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.mlp.c_proj", "P_bg1388", "P_bg1389", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.attn.in_proj_weight", "P_bg1388", "P_bg1389", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.attn.out_proj", "P_bg1388", "P_bg1389", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.mlp.c_fc", "P_bg1392", "P_bg1393", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.mlp.c_proj", "P_bg1394", "P_bg1395", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.attn.in_proj_weight", "P_bg1394", "P_bg1395", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.attn.out_proj", "P_bg1394", "P_bg1395", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.mlp.c_fc", "P_bg1398", "P_bg1399", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.mlp.c_proj", "P_bg1400", "P_bg1401", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.attn.in_proj_weight", "P_bg1400", "P_bg1401", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.attn.out_proj", "P_bg1400", "P_bg1401", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.mlp.c_fc", "P_bg1404", "P_bg1405", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.mlp.c_proj", "P_bg1406", "P_bg1407", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.attn.in_proj_weight", "P_bg1406", "P_bg1407", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.attn.out_proj", "P_bg1406", "P_bg1407", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.mlp.c_fc", "P_bg1410", "P_bg1411", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.mlp.c_proj", "P_bg1412", "P_bg1413", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.attn.in_proj_weight", "P_bg1412", "P_bg1413", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.attn.out_proj", "P_bg1412", "P_bg1413", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.mlp.c_fc", "P_bg1416", "P_bg1417", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.mlp.c_proj", "P_bg1418", "P_bg1419", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.attn.in_proj_weight", "P_bg1418", "P_bg1419", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.attn.out_proj", "P_bg1418", "P_bg1419", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.mlp.c_fc", "P_bg1422", "P_bg1423", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.mlp.c_proj", "P_bg1424", "P_bg1425", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.attn.in_proj_weight", "P_bg1424", "P_bg1425", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.attn.out_proj", "P_bg1424", "P_bg1425", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.mlp.c_fc", "P_bg1428", "P_bg1429", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.mlp.c_proj", "P_bg1430", "P_bg1431", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.attn.in_proj_weight", "P_bg1430", "P_bg1431", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.attn.out_proj", "P_bg1430", "P_bg1431", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.mlp.c_fc", "P_bg1434", "P_bg1435", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.mlp.c_proj", "P_bg1436", "P_bg1437", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.attn.in_proj_weight", "P_bg1436", "P_bg1437", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.attn.out_proj", "P_bg1436", "P_bg1437", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.mlp.c_fc", "P_bg1440", "P_bg1441", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.mlp.c_proj", "P_bg1442", "P_bg1443", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.attn.in_proj_weight", "P_bg1442", "P_bg1443", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.attn.out_proj", "P_bg1442", "P_bg1443", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.mlp.c_fc", "P_bg1446", "P_bg1447", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.mlp.c_proj", "P_bg1448", "P_bg1449", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.attn.in_proj_weight", "P_bg1448", "P_bg1449", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.attn.out_proj", "P_bg1448", "P_bg1449", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.mlp.c_fc", "P_bg1452", "P_bg1453", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.mlp.c_proj", "P_bg1454", "P_bg1455", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.attn.in_proj_weight", "P_bg1454", "P_bg1455", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.attn.out_proj", "P_bg1454", "P_bg1455", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.mlp.c_fc", "P_bg1458", "P_bg1459", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.mlp.c_proj", "P_bg1460", "P_bg1461", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.attn.in_proj_weight", "P_bg1460", "P_bg1461", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.attn.out_proj", "P_bg1460", "P_bg1461", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.mlp.c_fc", "P_bg1464", "P_bg1465", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.mlp.c_proj", "P_bg1466", "P_bg1467", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.embeddings.position_embedding", "P_bg1170", "P_bg1171", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.embeddings.token_embedding", "P_bg1172", "P_bg1173", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.mlp.fc1", "P_bg1176", "P_bg1177", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.mlp.fc2", "P_bg1178", "P_bg1179", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.k_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.out_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.q_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.v_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.mlp.fc1", "P_bg1184", "P_bg1185", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.mlp.fc2", "P_bg1186", "P_bg1187", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.k_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.out_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.q_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.v_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.mlp.fc1", "P_bg1192", "P_bg1193", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.mlp.fc2", "P_bg1194", "P_bg1195", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.k_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.out_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.q_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.v_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.mlp.fc1", "P_bg1200", "P_bg1201", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.mlp.fc2", "P_bg1202", "P_bg1203", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.k_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.out_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.q_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.v_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.mlp.fc1", "P_bg1208", "P_bg1209", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.mlp.fc2", "P_bg1210", "P_bg1211", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.k_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.out_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.q_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.v_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.mlp.fc1", "P_bg1216", "P_bg1217", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.mlp.fc2", "P_bg1218", "P_bg1219", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.k_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.out_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.q_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.v_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.mlp.fc1", "P_bg1224", "P_bg1225", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.mlp.fc2", "P_bg1226", "P_bg1227", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.k_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.out_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.q_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.v_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.mlp.fc1", "P_bg1232", "P_bg1233", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.mlp.fc2", "P_bg1234", "P_bg1235", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.k_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.out_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.q_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.v_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.mlp.fc1", "P_bg1240", "P_bg1241", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.mlp.fc2", "P_bg1242", "P_bg1243", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.k_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.out_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.q_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.v_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.mlp.fc1", "P_bg1248", "P_bg1249", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.mlp.fc2", "P_bg1250", "P_bg1251", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.k_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.out_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.q_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.v_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.mlp.fc1", "P_bg1256", "P_bg1257", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.mlp.fc2", "P_bg1258", "P_bg1259", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.k_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.out_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.q_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.v_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.mlp.fc1", "P_bg1264", "P_bg1265", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.mlp.fc2", "P_bg1266", "P_bg1267", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.k_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.out_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.q_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.v_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.1.model.token_embedding", "P_bg1272", "P_bg1273", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.attn.in_proj_weight", "P_bg1274", "P_bg1275", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.attn.out_proj", "P_bg1274", "P_bg1275", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.mlp.c_fc", "P_bg1278", "P_bg1279", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.mlp.c_proj", "P_bg1280", "P_bg1281", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.attn.in_proj_weight", "P_bg1280", "P_bg1281", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.attn.out_proj", "P_bg1280", "P_bg1281", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.mlp.c_fc", "P_bg1284", "P_bg1285", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.mlp.c_proj", "P_bg1286", "P_bg1287", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.attn.in_proj_weight", "P_bg1286", "P_bg1287", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.attn.out_proj", "P_bg1286", "P_bg1287", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.mlp.c_fc", "P_bg1290", "P_bg1291", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.mlp.c_proj", "P_bg1292", "P_bg1293", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.attn.in_proj_weight", "P_bg1292", "P_bg1293", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.attn.out_proj", "P_bg1292", "P_bg1293", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.mlp.c_fc", "P_bg1296", "P_bg1297", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.mlp.c_proj", "P_bg1298", "P_bg1299", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.attn.in_proj_weight", "P_bg1298", "P_bg1299", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.attn.out_proj", "P_bg1298", "P_bg1299", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.mlp.c_fc", "P_bg1302", "P_bg1303", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.mlp.c_proj", "P_bg1304", "P_bg1305", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.attn.in_proj_weight", "P_bg1304", "P_bg1305", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.attn.out_proj", "P_bg1304", "P_bg1305", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.mlp.c_fc", "P_bg1308", "P_bg1309", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.mlp.c_proj", "P_bg1310", "P_bg1311", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.attn.in_proj_weight", "P_bg1310", "P_bg1311", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.attn.out_proj", "P_bg1310", "P_bg1311", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.mlp.c_fc", "P_bg1314", "P_bg1315", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.mlp.c_proj", "P_bg1316", "P_bg1317", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.attn.in_proj_weight", "P_bg1316", "P_bg1317", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.attn.out_proj", "P_bg1316", "P_bg1317", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.mlp.c_fc", "P_bg1320", "P_bg1321", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.mlp.c_proj", "P_bg1322", "P_bg1323", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.attn.in_proj_weight", "P_bg1322", "P_bg1323", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.attn.out_proj", "P_bg1322", "P_bg1323", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.mlp.c_fc", "P_bg1326", "P_bg1327", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.mlp.c_proj", "P_bg1328", "P_bg1329", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.attn.in_proj_weight", "P_bg1328", "P_bg1329", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.attn.out_proj", "P_bg1328", "P_bg1329", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.mlp.c_fc", "P_bg1332", "P_bg1333", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.mlp.c_proj", "P_bg1334", "P_bg1335", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.attn.in_proj_weight", "P_bg1334", "P_bg1335", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.attn.out_proj", "P_bg1334", "P_bg1335", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.mlp.c_fc", "P_bg1338", "P_bg1339", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.mlp.c_proj", "P_bg1340", "P_bg1341", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.attn.in_proj_weight", "P_bg1340", "P_bg1341", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.attn.out_proj", "P_bg1340", "P_bg1341", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.mlp.c_fc", "P_bg1344", "P_bg1345", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.mlp.c_proj", "P_bg1346", "P_bg1347", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.attn.in_proj_weight", "P_bg1346", "P_bg1347", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.attn.out_proj", "P_bg1346", "P_bg1347", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.mlp.c_fc", "P_bg1350", "P_bg1351", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.mlp.c_proj", "P_bg1352", "P_bg1353", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.attn.in_proj_weight", "P_bg1352", "P_bg1353", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.attn.out_proj", "P_bg1352", "P_bg1353", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.mlp.c_fc", "P_bg1356", "P_bg1357", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.mlp.c_proj", "P_bg1358", "P_bg1359", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.attn.in_proj_weight", "P_bg1358", "P_bg1359", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.attn.out_proj", "P_bg1358", "P_bg1359", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.mlp.c_fc", "P_bg1362", "P_bg1363", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.mlp.c_proj", "P_bg1364", "P_bg1365", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.attn.in_proj_weight", "P_bg1364", "P_bg1365", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.attn.out_proj", "P_bg1364", "P_bg1365", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.mlp.c_fc", "P_bg1368", "P_bg1369", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.mlp.c_proj", "P_bg1370", "P_bg1371", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.attn.in_proj_weight", "P_bg1370", "P_bg1371", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.attn.out_proj", "P_bg1370", "P_bg1371", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.mlp.c_fc", "P_bg1374", "P_bg1375", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.mlp.c_proj", "P_bg1376", "P_bg1377", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.attn.in_proj_weight", "P_bg1376", "P_bg1377", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.attn.out_proj", "P_bg1376", "P_bg1377", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.mlp.c_fc", "P_bg1380", "P_bg1381", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.mlp.c_proj", "P_bg1382", "P_bg1383", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.attn.in_proj_weight", "P_bg1382", "P_bg1383", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.attn.out_proj", "P_bg1382", "P_bg1383", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.mlp.c_fc", "P_bg1386", "P_bg1387", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.mlp.c_proj", "P_bg1388", "P_bg1389", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.attn.in_proj_weight", "P_bg1388", "P_bg1389", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.attn.out_proj", "P_bg1388", "P_bg1389", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.mlp.c_fc", "P_bg1392", "P_bg1393", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.mlp.c_proj", "P_bg1394", "P_bg1395", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.attn.in_proj_weight", "P_bg1394", "P_bg1395", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.attn.out_proj", "P_bg1394", "P_bg1395", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.mlp.c_fc", "P_bg1398", "P_bg1399", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.mlp.c_proj", "P_bg1400", "P_bg1401", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.attn.in_proj_weight", "P_bg1400", "P_bg1401", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.attn.out_proj", "P_bg1400", "P_bg1401", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.mlp.c_fc", "P_bg1404", "P_bg1405", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.mlp.c_proj", "P_bg1406", "P_bg1407", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.attn.in_proj_weight", "P_bg1406", "P_bg1407", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.attn.out_proj", "P_bg1406", "P_bg1407", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.mlp.c_fc", "P_bg1410", "P_bg1411", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.mlp.c_proj", "P_bg1412", "P_bg1413", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.attn.in_proj_weight", "P_bg1412", "P_bg1413", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.attn.out_proj", "P_bg1412", "P_bg1413", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.mlp.c_fc", "P_bg1416", "P_bg1417", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.mlp.c_proj", "P_bg1418", "P_bg1419", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.attn.in_proj_weight", "P_bg1418", "P_bg1419", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.attn.out_proj", "P_bg1418", "P_bg1419", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.mlp.c_fc", "P_bg1422", "P_bg1423", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.mlp.c_proj", "P_bg1424", "P_bg1425", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.attn.in_proj_weight", "P_bg1424", "P_bg1425", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.attn.out_proj", "P_bg1424", "P_bg1425", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.mlp.c_fc", "P_bg1428", "P_bg1429", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.mlp.c_proj", "P_bg1430", "P_bg1431", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.attn.in_proj_weight", "P_bg1430", "P_bg1431", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.attn.out_proj", "P_bg1430", "P_bg1431", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.mlp.c_fc", "P_bg1434", "P_bg1435", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.mlp.c_proj", "P_bg1436", "P_bg1437", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.attn.in_proj_weight", "P_bg1436", "P_bg1437", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.attn.out_proj", "P_bg1436", "P_bg1437", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.mlp.c_fc", "P_bg1440", "P_bg1441", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.mlp.c_proj", "P_bg1442", "P_bg1443", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.attn.in_proj_weight", "P_bg1442", "P_bg1443", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.attn.out_proj", "P_bg1442", "P_bg1443", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.mlp.c_fc", "P_bg1446", "P_bg1447", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.mlp.c_proj", "P_bg1448", "P_bg1449", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.attn.in_proj_weight", "P_bg1448", "P_bg1449", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.attn.out_proj", "P_bg1448", "P_bg1449", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.mlp.c_fc", "P_bg1452", "P_bg1453", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.mlp.c_proj", "P_bg1454", "P_bg1455", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.attn.in_proj_weight", "P_bg1454", "P_bg1455", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.attn.out_proj", "P_bg1454", "P_bg1455", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.mlp.c_fc", "P_bg1458", "P_bg1459", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.mlp.c_proj", "P_bg1460", "P_bg1461", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.attn.in_proj_weight", "P_bg1460", "P_bg1461", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.attn.out_proj", "P_bg1460", "P_bg1461", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.mlp.c_fc", "P_bg1464", "P_bg1465", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.mlp.c_proj", "P_bg1466", "P_bg1467", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.embeddings.position_embedding", "P_bg1170", "P_bg1171", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.embeddings.token_embedding", "P_bg1172", "P_bg1173", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.mlp.fc1", "P_bg1176", "P_bg1177", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.mlp.fc2", "P_bg1178", "P_bg1179", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.k_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.out_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.q_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.0.self_attn.v_proj", "P_bg1180", "P_bg1181", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.mlp.fc1", "P_bg1184", "P_bg1185", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.mlp.fc2", "P_bg1186", "P_bg1187", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.k_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.out_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.q_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.1.self_attn.v_proj", "P_bg1188", "P_bg1189", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.mlp.fc1", "P_bg1192", "P_bg1193", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.mlp.fc2", "P_bg1194", "P_bg1195", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.k_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.out_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.q_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.10.self_attn.v_proj", "P_bg1196", "P_bg1197", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.mlp.fc1", "P_bg1200", "P_bg1201", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.mlp.fc2", "P_bg1202", "P_bg1203", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.k_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.out_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.q_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.11.self_attn.v_proj", "P_bg1204", "P_bg1205", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.mlp.fc1", "P_bg1208", "P_bg1209", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.mlp.fc2", "P_bg1210", "P_bg1211", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.k_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.out_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.q_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.2.self_attn.v_proj", "P_bg1212", "P_bg1213", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.mlp.fc1", "P_bg1216", "P_bg1217", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.mlp.fc2", "P_bg1218", "P_bg1219", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.k_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.out_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.q_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.3.self_attn.v_proj", "P_bg1220", "P_bg1221", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.mlp.fc1", "P_bg1224", "P_bg1225", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.mlp.fc2", "P_bg1226", "P_bg1227", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.k_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.out_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.q_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.4.self_attn.v_proj", "P_bg1228", "P_bg1229", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.mlp.fc1", "P_bg1232", "P_bg1233", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.mlp.fc2", "P_bg1234", "P_bg1235", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.k_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.out_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.q_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.5.self_attn.v_proj", "P_bg1236", "P_bg1237", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.mlp.fc1", "P_bg1240", "P_bg1241", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.mlp.fc2", "P_bg1242", "P_bg1243", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.k_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.out_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.q_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.6.self_attn.v_proj", "P_bg1244", "P_bg1245", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.mlp.fc1", "P_bg1248", "P_bg1249", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.mlp.fc2", "P_bg1250", "P_bg1251", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.k_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.out_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.q_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.7.self_attn.v_proj", "P_bg1252", "P_bg1253", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.mlp.fc1", "P_bg1256", "P_bg1257", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.mlp.fc2", "P_bg1258", "P_bg1259", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.k_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.out_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.q_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.8.self_attn.v_proj", "P_bg1260", "P_bg1261", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.mlp.fc1", "P_bg1264", "P_bg1265", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.mlp.fc2", "P_bg1266", "P_bg1267", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.k_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.out_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.q_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.0.transformer.text_model.encoder.layers.9.self_attn.v_proj", "P_bg1268", "P_bg1269", bias=False), + **dense("conditioner.embedders.1.model.token_embedding", "P_bg1272", "P_bg1273", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.attn.in_proj_weight", "P_bg1274", "P_bg1275", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.attn.out_proj", "P_bg1274", "P_bg1275", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.mlp.c_fc", "P_bg1278", "P_bg1279", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.0.mlp.c_proj", "P_bg1280", "P_bg1281", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.attn.in_proj_weight", "P_bg1280", "P_bg1281", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.attn.out_proj", "P_bg1280", "P_bg1281", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.mlp.c_fc", "P_bg1284", "P_bg1285", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.1.mlp.c_proj", "P_bg1286", "P_bg1287", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.attn.in_proj_weight", "P_bg1286", "P_bg1287", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.attn.out_proj", "P_bg1286", "P_bg1287", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.mlp.c_fc", "P_bg1290", "P_bg1291", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.10.mlp.c_proj", "P_bg1292", "P_bg1293", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.attn.in_proj_weight", "P_bg1292", "P_bg1293", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.attn.out_proj", "P_bg1292", "P_bg1293", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.mlp.c_fc", "P_bg1296", "P_bg1297", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.11.mlp.c_proj", "P_bg1298", "P_bg1299", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.attn.in_proj_weight", "P_bg1298", "P_bg1299", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.attn.out_proj", "P_bg1298", "P_bg1299", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.mlp.c_fc", "P_bg1302", "P_bg1303", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.12.mlp.c_proj", "P_bg1304", "P_bg1305", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.attn.in_proj_weight", "P_bg1304", "P_bg1305", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.attn.out_proj", "P_bg1304", "P_bg1305", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.mlp.c_fc", "P_bg1308", "P_bg1309", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.13.mlp.c_proj", "P_bg1310", "P_bg1311", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.attn.in_proj_weight", "P_bg1310", "P_bg1311", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.attn.out_proj", "P_bg1310", "P_bg1311", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.mlp.c_fc", "P_bg1314", "P_bg1315", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.14.mlp.c_proj", "P_bg1316", "P_bg1317", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.attn.in_proj_weight", "P_bg1316", "P_bg1317", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.attn.out_proj", "P_bg1316", "P_bg1317", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.mlp.c_fc", "P_bg1320", "P_bg1321", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.15.mlp.c_proj", "P_bg1322", "P_bg1323", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.attn.in_proj_weight", "P_bg1322", "P_bg1323", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.attn.out_proj", "P_bg1322", "P_bg1323", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.mlp.c_fc", "P_bg1326", "P_bg1327", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.16.mlp.c_proj", "P_bg1328", "P_bg1329", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.attn.in_proj_weight", "P_bg1328", "P_bg1329", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.attn.out_proj", "P_bg1328", "P_bg1329", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.mlp.c_fc", "P_bg1332", "P_bg1333", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.17.mlp.c_proj", "P_bg1334", "P_bg1335", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.attn.in_proj_weight", "P_bg1334", "P_bg1335", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.attn.out_proj", "P_bg1334", "P_bg1335", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.mlp.c_fc", "P_bg1338", "P_bg1339", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.18.mlp.c_proj", "P_bg1340", "P_bg1341", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.attn.in_proj_weight", "P_bg1340", "P_bg1341", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.attn.out_proj", "P_bg1340", "P_bg1341", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.mlp.c_fc", "P_bg1344", "P_bg1345", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.19.mlp.c_proj", "P_bg1346", "P_bg1347", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.attn.in_proj_weight", "P_bg1346", "P_bg1347", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.attn.out_proj", "P_bg1346", "P_bg1347", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.mlp.c_fc", "P_bg1350", "P_bg1351", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.2.mlp.c_proj", "P_bg1352", "P_bg1353", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.attn.in_proj_weight", "P_bg1352", "P_bg1353", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.attn.out_proj", "P_bg1352", "P_bg1353", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.mlp.c_fc", "P_bg1356", "P_bg1357", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.20.mlp.c_proj", "P_bg1358", "P_bg1359", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.attn.in_proj_weight", "P_bg1358", "P_bg1359", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.attn.out_proj", "P_bg1358", "P_bg1359", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.mlp.c_fc", "P_bg1362", "P_bg1363", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.21.mlp.c_proj", "P_bg1364", "P_bg1365", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.attn.in_proj_weight", "P_bg1364", "P_bg1365", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.attn.out_proj", "P_bg1364", "P_bg1365", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.mlp.c_fc", "P_bg1368", "P_bg1369", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.22.mlp.c_proj", "P_bg1370", "P_bg1371", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.attn.in_proj_weight", "P_bg1370", "P_bg1371", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.attn.out_proj", "P_bg1370", "P_bg1371", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.mlp.c_fc", "P_bg1374", "P_bg1375", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.23.mlp.c_proj", "P_bg1376", "P_bg1377", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.attn.in_proj_weight", "P_bg1376", "P_bg1377", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.attn.out_proj", "P_bg1376", "P_bg1377", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.mlp.c_fc", "P_bg1380", "P_bg1381", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.24.mlp.c_proj", "P_bg1382", "P_bg1383", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.attn.in_proj_weight", "P_bg1382", "P_bg1383", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.attn.out_proj", "P_bg1382", "P_bg1383", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.mlp.c_fc", "P_bg1386", "P_bg1387", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.25.mlp.c_proj", "P_bg1388", "P_bg1389", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.attn.in_proj_weight", "P_bg1388", "P_bg1389", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.attn.out_proj", "P_bg1388", "P_bg1389", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.mlp.c_fc", "P_bg1392", "P_bg1393", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.26.mlp.c_proj", "P_bg1394", "P_bg1395", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.attn.in_proj_weight", "P_bg1394", "P_bg1395", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.attn.out_proj", "P_bg1394", "P_bg1395", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.mlp.c_fc", "P_bg1398", "P_bg1399", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.27.mlp.c_proj", "P_bg1400", "P_bg1401", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.attn.in_proj_weight", "P_bg1400", "P_bg1401", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.attn.out_proj", "P_bg1400", "P_bg1401", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.mlp.c_fc", "P_bg1404", "P_bg1405", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.28.mlp.c_proj", "P_bg1406", "P_bg1407", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.attn.in_proj_weight", "P_bg1406", "P_bg1407", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.attn.out_proj", "P_bg1406", "P_bg1407", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.mlp.c_fc", "P_bg1410", "P_bg1411", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.29.mlp.c_proj", "P_bg1412", "P_bg1413", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.attn.in_proj_weight", "P_bg1412", "P_bg1413", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.attn.out_proj", "P_bg1412", "P_bg1413", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.mlp.c_fc", "P_bg1416", "P_bg1417", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.3.mlp.c_proj", "P_bg1418", "P_bg1419", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.attn.in_proj_weight", "P_bg1418", "P_bg1419", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.attn.out_proj", "P_bg1418", "P_bg1419", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.mlp.c_fc", "P_bg1422", "P_bg1423", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.30.mlp.c_proj", "P_bg1424", "P_bg1425", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.attn.in_proj_weight", "P_bg1424", "P_bg1425", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.attn.out_proj", "P_bg1424", "P_bg1425", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.mlp.c_fc", "P_bg1428", "P_bg1429", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.31.mlp.c_proj", "P_bg1430", "P_bg1431", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.attn.in_proj_weight", "P_bg1430", "P_bg1431", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.attn.out_proj", "P_bg1430", "P_bg1431", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.mlp.c_fc", "P_bg1434", "P_bg1435", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.4.mlp.c_proj", "P_bg1436", "P_bg1437", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.attn.in_proj_weight", "P_bg1436", "P_bg1437", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.attn.out_proj", "P_bg1436", "P_bg1437", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.mlp.c_fc", "P_bg1440", "P_bg1441", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.5.mlp.c_proj", "P_bg1442", "P_bg1443", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.attn.in_proj_weight", "P_bg1442", "P_bg1443", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.attn.out_proj", "P_bg1442", "P_bg1443", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.mlp.c_fc", "P_bg1446", "P_bg1447", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.6.mlp.c_proj", "P_bg1448", "P_bg1449", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.attn.in_proj_weight", "P_bg1448", "P_bg1449", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.attn.out_proj", "P_bg1448", "P_bg1449", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.mlp.c_fc", "P_bg1452", "P_bg1453", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.7.mlp.c_proj", "P_bg1454", "P_bg1455", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.attn.in_proj_weight", "P_bg1454", "P_bg1455", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.attn.out_proj", "P_bg1454", "P_bg1455", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.mlp.c_fc", "P_bg1458", "P_bg1459", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.8.mlp.c_proj", "P_bg1460", "P_bg1461", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.attn.in_proj_weight", "P_bg1460", "P_bg1461", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.attn.out_proj", "P_bg1460", "P_bg1461", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.mlp.c_fc", "P_bg1464", "P_bg1465", bias=False), + **dense("conditioner.embedders.1.model.transformer.resblocks.9.mlp.c_proj", "P_bg1466", "P_bg1467", bias=False), + + } + ) diff --git a/modules/merging/merge_rebasin.py b/modules/merging/merge_rebasin.py index dd8d28b44..d5b6e4fdc 100644 --- a/modules/merging/merge_rebasin.py +++ b/modules/merging/merge_rebasin.py @@ -6,6 +6,7 @@ import torch from scipy.optimize import linear_sum_assignment from modules.shared import log + SPECIAL_KEYS = [ "first_stage_model.decoder.norm_out.weight", "first_stage_model.decoder.norm_out.bias", @@ -30,2044 +31,6 @@ def permutation_spec_from_axes_to_perm(axes_to_perm: dict) -> PermutationSpec: return PermutationSpec(perm_to_axes=dict(perm_to_axes), axes_to_perm=axes_to_perm) -def sdunet_permutation_spec() -> PermutationSpec: - conv = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment - f"{name}.weight": ( - p_out, - p_in, - ), - f"{name}.bias": (p_out,), - } - norm = lambda name, p: {f"{name}.weight": (p,), f"{name}.bias": (p,)} # pylint: disable=unnecessary-lambda-assignment - dense = ( - lambda name, p_in, p_out, bias=True: { # pylint: disable=unnecessary-lambda-assignment - f"{name}.weight": (p_out, p_in), - f"{name}.bias": (p_out,), - } - if bias - else {f"{name}.weight": (p_out, p_in)} - ) - skip = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment - f"{name}": ( - p_out, - p_in, - None, - None, - ) - } - - # Unet Res blocks - easyblock = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment - **norm(f"{name}.in_layers.0", p_in), - **conv(f"{name}.in_layers.2", p_in, f"P_{name}_inner"), - **dense( - f"{name}.emb_layers.1", f"P_{name}_inner2", f"P_{name}_inner3", bias=True - ), - **norm(f"{name}.out_layers.0", f"P_{name}_inner4"), - **conv(f"{name}.out_layers.3", f"P_{name}_inner4", p_out), - } - - # VAE blocks - Unused - easyblock2 = lambda name, p: { # pylint: disable=unnecessary-lambda-assignment - **norm(f"{name}.norm1", p), - **conv(f"{name}.conv1", p, f"P_{name}_inner"), - **norm(f"{name}.norm2", f"P_{name}_inner"), - **conv(f"{name}.conv2", f"P_{name}_inner", p), - } - - # This is for blocks that use a residual connection, but change the number of channels via a Conv. - shortcutblock = lambda name, p_in, p_out: { # pylint: disable=unnecessary-lambda-assignment - **norm(f"{name}.norm1", p_in), - **conv(f"{name}.conv1", p_in, f"P_{name}_inner"), - **norm(f"{name}.norm2", f"P_{name}_inner"), - **conv(f"{name}.conv2", f"P_{name}_inner", p_out), - **conv(f"{name}.nin_shortcut", p_in, p_out), - **norm(f"{name}.nin_shortcut", p_out), - } - - return permutation_spec_from_axes_to_perm( - { - # Skipped Layers - **skip("betas", None, None), - **skip("alphas_cumprod", None, None), - **skip("alphas_cumprod_prev", None, None), - **skip("sqrt_alphas_cumprod", None, None), - **skip("sqrt_one_minus_alphas_cumprod", None, None), - **skip("log_one_minus_alphas_cumprods", None, None), - **skip("sqrt_recip_alphas_cumprod", None, None), - **skip("sqrt_recipm1_alphas_cumprod", None, None), - **skip("posterior_variance", None, None), - **skip("posterior_log_variance_clipped", None, None), - **skip("posterior_mean_coef1", None, None), - **skip("posterior_mean_coef2", None, None), - **skip("log_one_minus_alphas_cumprod", None, None), - **skip("model_ema.decay", None, None), - **skip("model_ema.num_updates", None, None), - # initial - **dense("model.diffusion_model.time_embed.0", None, "P_bg0", bias=True), - **dense("model.diffusion_model.time_embed.2", "P_bg0", "P_bg1", bias=True), - **conv("model.diffusion_model.input_blocks.0.0", "P_bg2", "P_bg3"), - # input blocks - **easyblock("model.diffusion_model.input_blocks.1.0", "P_bg4", "P_bg5"), - **norm("model.diffusion_model.input_blocks.1.1.norm", "P_bg6"), - **conv("model.diffusion_model.input_blocks.1.1.proj_in", "P_bg6", "P_bg7"), - **dense( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_q", - "P_bg8", - "P_bg9", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_k", - "P_bg8", - "P_bg9", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_v", - "P_bg8", - "P_bg9", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_out.0", - "P_bg8", - "P_bg9", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.ff.net.0.proj", - "P_bg10", - "P_bg11", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.ff.net.2", - "P_bg12", - "P_bg13", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_q", - "P_bg14", - "P_bg15", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_k", - "P_bg16", - "P_bg17", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_v", - "P_bg16", - "P_bg17", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_out.0", - "P_bg18", - "P_bg19", - bias=True, - ), - **norm( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.norm1", - "P_bg19", - ), - **norm( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.norm2", - "P_bg19", - ), - **norm( - "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.norm3", - "P_bg19", - ), - **conv( - "model.diffusion_model.input_blocks.1.1.proj_out", "P_bg19", "P_bg20" - ), - **easyblock("model.diffusion_model.input_blocks.2.0", "P_bg21", "P_bg22"), - **norm("model.diffusion_model.input_blocks.2.1.norm", "P_bg23"), - **conv( - "model.diffusion_model.input_blocks.2.1.proj_in", "P_bg23", "P_bg24" - ), - **dense( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn1.to_q", - "P_bg25", - "P_bg26", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn1.to_k", - "P_bg25", - "P_bg26", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn1.to_v", - "P_bg25", - "P_bg26", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn1.to_out.0", - "P_bg25", - "P_bg26", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.ff.net.0.proj", - "P_bg27", - "P_bg28", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.ff.net.2", - "P_bg29", - "P_bg30", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_q", - "P_bg31", - "P_bg32", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_k", - "P_bg33", - "P_bg34", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_v", - "P_bg33", - "P_bg34", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_out.0", - "P_bg35", - "P_bg36", - bias=True, - ), - **norm( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.norm1", - "P_bg36", - ), - **norm( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.norm2", - "P_bg36", - ), - **norm( - "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.norm3", - "P_bg36", - ), - **conv( - "model.diffusion_model.input_blocks.2.1.proj_out", "P_bg36", "P_bg37" - ), - **conv("model.diffusion_model.input_blocks.3.0.op", "P_bg38", "P_bg39"), - **easyblock("model.diffusion_model.input_blocks.4.0", "P_bg40", "P_bg41"), - **conv( - "model.diffusion_model.input_blocks.4.0.skip_connection", - "P_bg42", - "P_bg43", - ), - **norm("model.diffusion_model.input_blocks.4.1.norm", "P_bg44"), - **conv( - "model.diffusion_model.input_blocks.4.1.proj_in", "P_bg44", "P_bg45" - ), - **dense( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_q", - "P_bg46", - "P_bg47", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_k", - "P_bg46", - "P_bg47", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_v", - "P_bg46", - "P_bg47", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn1.to_out.0", - "P_bg46", - "P_bg47", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.ff.net.0.proj", - "P_bg48", - "P_bg49", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.ff.net.2", - "P_bg50", - "P_bg51", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_q", - "P_bg52", - "P_bg53", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_k", - "P_bg54", - "P_bg55", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_v", - "P_bg54", - "P_bg55", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_out.0", - "P_bg56", - "P_bg57", - bias=True, - ), - **norm( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.norm1", - "P_bg57", - ), - **norm( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.norm2", - "P_bg57", - ), - **norm( - "model.diffusion_model.input_blocks.4.1.transformer_blocks.0.norm3", - "P_bg57", - ), - **conv( - "model.diffusion_model.input_blocks.4.1.proj_out", "P_bg57", "P_bg58" - ), - **easyblock("model.diffusion_model.input_blocks.5.0", "P_bg59", "P_bg60"), - **norm("model.diffusion_model.input_blocks.5.1.norm", "P_bg61"), - **conv( - "model.diffusion_model.input_blocks.5.1.proj_in", "P_bg61", "P_bg62" - ), - **dense( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_q", - "P_bg63", - "P_bg64", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_k", - "P_bg63", - "P_bg64", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_v", - "P_bg63", - "P_bg64", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn1.to_out.0", - "P_bg63", - "P_bg64", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.ff.net.0.proj", - "P_bg65", - "P_bg66", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.ff.net.2", - "P_bg67", - "P_bg68", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_q", - "P_bg69", - "P_bg70", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_k", - "P_bg71", - "P_bg72", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_v", - "P_bg71", - "P_bg72", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.attn2.to_out.0", - "P_bg73", - "P_bg74", - bias=True, - ), - **norm( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.norm1", - "P_bg74", - ), - **norm( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.norm2", - "P_bg74", - ), - **norm( - "model.diffusion_model.input_blocks.5.1.transformer_blocks.0.norm3", - "P_bg74", - ), - **conv( - "model.diffusion_model.input_blocks.5.1.proj_out", "P_bg74", "P_bg75" - ), - **conv("model.diffusion_model.input_blocks.6.0.op", "P_bg76", "P_bg77"), - **easyblock("model.diffusion_model.input_blocks.7.0", "P_bg78", "P_bg79"), - **conv( - "model.diffusion_model.input_blocks.7.0.skip_connection", - "P_bg80", - "P_bg81", - ), - **norm("model.diffusion_model.input_blocks.7.1.norm", "P_bg82"), - **conv( - "model.diffusion_model.input_blocks.7.1.proj_in", "P_bg82", "P_bg83" - ), - **dense( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_q", - "P_bg84", - "P_bg85", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_k", - "P_bg84", - "P_bg85", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_v", - "P_bg84", - "P_bg85", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn1.to_out.0", - "P_bg84", - "P_bg85", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.ff.net.0.proj", - "P_bg86", - "P_bg87", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.ff.net.2", - "P_bg88", - "P_bg89", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_q", - "P_bg90", - "P_bg91", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_k", - "P_bg92", - "P_bg93", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_v", - "P_bg92", - "P_bg93", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.attn2.to_out.0", - "P_bg94", - "P_bg95", - bias=True, - ), - **norm( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.norm1", - "P_bg95", - ), - **norm( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.norm2", - "P_bg95", - ), - **norm( - "model.diffusion_model.input_blocks.7.1.transformer_blocks.0.norm3", - "P_bg95", - ), - **conv( - "model.diffusion_model.input_blocks.7.1.proj_out", "P_bg95", "P_bg96" - ), - **easyblock("model.diffusion_model.input_blocks.8.0", "P_bg97", "P_bg98"), - **norm("model.diffusion_model.input_blocks.8.1.norm", "P_bg99"), - **conv( - "model.diffusion_model.input_blocks.8.1.proj_in", "P_bg99", "P_bg100" - ), - **dense( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_q", - "P_bg101", - "P_bg102", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_k", - "P_bg101", - "P_bg102", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_v", - "P_bg101", - "P_bg102", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn1.to_out.0", - "P_bg101", - "P_bg102", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.ff.net.0.proj", - "P_bg103", - "P_bg104", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.ff.net.2", - "P_bg105", - "P_bg106", - bias=True, - ), - **dense( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_q", - "P_bg107", - "P_bg108", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_k", - "P_bg109", - "P_bg110", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_v", - "P_bg109", - "P_bg110", - bias=False, - ), - **dense( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.attn2.to_out.0", - "P_bg111", - "P_bg112", - bias=True, - ), - **norm( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.norm1", - "P_bg112", - ), - **norm( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.norm2", - "P_bg112", - ), - **norm( - "model.diffusion_model.input_blocks.8.1.transformer_blocks.0.norm3", - "P_bg112", - ), - **conv( - "model.diffusion_model.input_blocks.8.1.proj_out", "P_bg112", "P_bg113" - ), - **conv("model.diffusion_model.input_blocks.9.0.op", "P_bg114", "P_bg115"), - **easyblock( - "model.diffusion_model.input_blocks.10.0", "P_bg115", "P_bg116" - ), - **easyblock( - "model.diffusion_model.input_blocks.11.0", "P_bg116", "P_bg117" - ), - # middle blocks - **easyblock("model.diffusion_model.middle_block.0", "P_bg117", "P_bg118"), - **norm("model.diffusion_model.middle_block.1.norm", "P_bg119"), - **conv( - "model.diffusion_model.middle_block.1.proj_in", "P_bg119", "P_bg120" - ), - **dense( - "model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_q", - "P_bg121", - "P_bg122", - bias=False, - ), - **dense( - "model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_k", - "P_bg121", - "P_bg122", - bias=False, - ), - **dense( - "model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_v", - "P_bg121", - "P_bg122", - bias=False, - ), - **dense( - "model.diffusion_model.middle_block.1.transformer_blocks.0.attn1.to_out.0", - "P_bg121", - "P_bg122", - bias=True, - ), - **dense( - "model.diffusion_model.middle_block.1.transformer_blocks.0.ff.net.0.proj", - "P_bg123", - "P_bg124", - bias=True, - ), - **dense( - "model.diffusion_model.middle_block.1.transformer_blocks.0.ff.net.2", - "P_bg125", - "P_bg126", - bias=True, - ), - **dense( - "model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_q", - "P_bg127", - "P_bg128", - bias=False, - ), - **dense( - "model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_k", - "P_bg129", - "P_bg130", - bias=False, - ), - **dense( - "model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_v", - "P_bg129", - "P_bg130", - bias=False, - ), - **dense( - "model.diffusion_model.middle_block.1.transformer_blocks.0.attn2.to_out.0", - "P_bg131", - "P_bg132", - bias=True, - ), - **norm( - "model.diffusion_model.middle_block.1.transformer_blocks.0.norm1", - "P_bg132", - ), - **norm( - "model.diffusion_model.middle_block.1.transformer_blocks.0.norm2", - "P_bg132", - ), - **norm( - "model.diffusion_model.middle_block.1.transformer_blocks.0.norm3", - "P_bg132", - ), - **conv( - "model.diffusion_model.middle_block.1.proj_out", "P_bg132", "P_bg133" - ), - **easyblock("model.diffusion_model.middle_block.2", "P_bg134", "P_bg135"), - # output blocks - **easyblock( - "model.diffusion_model.output_blocks.0.0", "P_bg136", "P_bg137" - ), - **conv( - "model.diffusion_model.output_blocks.0.0.skip_connection", - "P_bg138", - "P_bg139", - ), - **easyblock( - "model.diffusion_model.output_blocks.1.0", "P_bg140", "P_bg141" - ), - **conv( - "model.diffusion_model.output_blocks.1.0.skip_connection", - "P_bg142", - "P_bg143", - ), - **easyblock( - "model.diffusion_model.output_blocks.2.0", "P_bg144", "P_bg145" - ), - **conv( - "model.diffusion_model.output_blocks.2.0.skip_connection", - "P_bg146", - "P_bg147", - ), - **conv( - "model.diffusion_model.output_blocks.2.1.conv", "P_bg148", "P_bg149" - ), - **easyblock( - "model.diffusion_model.output_blocks.3.0", "P_bg150", "P_bg151" - ), - **conv( - "model.diffusion_model.output_blocks.3.0.skip_connection", - "P_bg152", - "P_bg153", - ), - **norm("model.diffusion_model.output_blocks.3.1.norm", "P_bg154"), - **conv( - "model.diffusion_model.output_blocks.3.1.proj_in", "P_bg154", "P_bg155" - ), - **dense( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_q", - "P_bg156", - "P_bg157", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_k", - "P_bg156", - "P_bg157", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_v", - "P_bg156", - "P_bg157", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn1.to_out.0", - "P_bg156", - "P_bg157", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.ff.net.0.proj", - "P_bg158", - "P_bg159", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.ff.net.2", - "P_bg160", - "P_bg161", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_q", - "P_bg162", - "P_bg163", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_k", - "P_bg164", - "P_bg165", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_v", - "P_bg164", - "P_bg165", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.attn2.to_out.0", - "P_bg166", - "P_bg167", - bias=True, - ), - **norm( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.norm1", - "P_bg167", - ), - **norm( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.norm2", - "P_bg167", - ), - **norm( - "model.diffusion_model.output_blocks.3.1.transformer_blocks.0.norm3", - "P_bg167", - ), - **conv( - "model.diffusion_model.output_blocks.3.1.proj_out", "P_bg167", "P_bg168" - ), - **easyblock( - "model.diffusion_model.output_blocks.4.0", "P_bg169", "P_bg170" - ), - **conv( - "model.diffusion_model.output_blocks.4.0.skip_connection", - "P_bg171", - "P_bg172", - ), - **norm("model.diffusion_model.output_blocks.4.1.norm", "P_bg173"), - **conv( - "model.diffusion_model.output_blocks.4.1.proj_in", "P_bg173", "P_bg174" - ), - **dense( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_q", - "P_bg175", - "P_bg176", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_k", - "P_bg175", - "P_bg176", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_v", - "P_bg175", - "P_bg176", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn1.to_out.0", - "P_bg175", - "P_bg176", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.ff.net.0.proj", - "P_bg177", - "P_bg178", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.ff.net.2", - "P_bg179", - "P_bg180", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_q", - "P_bg181", - "P_bg182", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_k", - "P_bg183", - "P_bg184", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_v", - "P_bg183", - "P_bg184", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.attn2.to_out.0", - "P_bg185", - "P_bg186", - bias=True, - ), - **norm( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.norm1", - "P_bg186", - ), - **norm( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.norm2", - "P_bg186", - ), - **norm( - "model.diffusion_model.output_blocks.4.1.transformer_blocks.0.norm3", - "P_bg186", - ), - **conv( - "model.diffusion_model.output_blocks.4.1.proj_out", "P_bg186", "P_bg187" - ), - **easyblock( - "model.diffusion_model.output_blocks.5.0", "P_bg188", "P_bg189" - ), - **conv( - "model.diffusion_model.output_blocks.5.0.skip_connection", - "P_bg190", - "P_bg191", - ), - **norm("model.diffusion_model.output_blocks.5.1.norm", "P_bg192"), - **conv( - "model.diffusion_model.output_blocks.5.1.proj_in", "P_bg192", "P_bg193" - ), - **dense( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_q", - "P_bg194", - "P_bg195", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_k", - "P_bg194", - "P_bg195", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_v", - "P_bg194", - "P_bg195", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn1.to_out.0", - "P_bg194", - "P_bg195", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.ff.net.0.proj", - "P_bg196", - "P_bg197", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.ff.net.2", - "P_bg198", - "P_bg199", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_q", - "P_bg200", - "P_bg201", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_k", - "P_bg202", - "P_bg203", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_v", - "P_bg202", - "P_bg203", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.attn2.to_out.0", - "P_bg204", - "P_bg205", - bias=True, - ), - **norm( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.norm1", - "P_bg205", - ), - **norm( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.norm2", - "P_bg205", - ), - **norm( - "model.diffusion_model.output_blocks.5.1.transformer_blocks.0.norm3", - "P_bg205", - ), - **conv( - "model.diffusion_model.output_blocks.5.1.proj_out", "P_bg205", "P_bg206" - ), - **conv( - "model.diffusion_model.output_blocks.5.2.conv", "P_bg206", "P_bg207" - ), - **easyblock( - "model.diffusion_model.output_blocks.6.0", "P_bg208", "P_bg209" - ), - **conv( - "model.diffusion_model.output_blocks.6.0.skip_connection", - "P_bg210", - "P_bg211", - ), - **norm("model.diffusion_model.output_blocks.6.1.norm", "P_bg212"), - **conv( - "model.diffusion_model.output_blocks.6.1.proj_in", "P_bg212", "P_bg213" - ), - **dense( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn1.to_q", - "P_bg214", - "P_bg215", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn1.to_k", - "P_bg214", - "P_bg215", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn1.to_v", - "P_bg214", - "P_bg215", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn1.to_out.0", - "P_bg214", - "P_bg215", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.ff.net.0.proj", - "P_bg216", - "P_bg217", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.ff.net.2", - "P_bg218", - "P_bg219", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn2.to_q", - "P_bg220", - "P_bg221", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn2.to_k", - "P_bg222", - "P_bg223", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn2.to_v", - "P_bg222", - "P_bg223", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.attn2.to_out.0", - "P_bg224", - "P_bg225", - bias=True, - ), - **norm( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.norm1", - "P_bg225", - ), - **norm( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.norm2", - "P_bg225", - ), - **norm( - "model.diffusion_model.output_blocks.6.1.transformer_blocks.0.norm3", - "P_bg225", - ), - **conv( - "model.diffusion_model.output_blocks.6.1.proj_out", "P_bg225", "P_bg226" - ), - **easyblock( - "model.diffusion_model.output_blocks.7.0", "P_bg227", "P_bg228" - ), - **conv( - "model.diffusion_model.output_blocks.7.0.skip_connection", - "P_bg229", - "P_bg230", - ), - **norm("model.diffusion_model.output_blocks.7.1.norm", "P_bg231"), - **conv( - "model.diffusion_model.output_blocks.7.1.proj_in", "P_bg231", "P_bg232" - ), - **dense( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_q", - "P_bg233", - "P_bg234", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_k", - "P_bg233", - "P_bg234", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_v", - "P_bg233", - "P_bg234", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_out.0", - "P_bg233", - "P_bg234", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.ff.net.0.proj", - "P_bg235", - "P_bg236", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.ff.net.2", - "P_bg237", - "P_bg238", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn2.to_q", - "P_bg239", - "P_bg240", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn2.to_k", - "P_bg241", - "P_bg242", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn2.to_v", - "P_bg241", - "P_bg242", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn2.to_out.0", - "P_bg243", - "P_bg244", - bias=True, - ), - **norm( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.norm1", - "P_bg244", - ), - **norm( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.norm2", - "P_bg244", - ), - **norm( - "model.diffusion_model.output_blocks.7.1.transformer_blocks.0.norm3", - "P_bg244", - ), - **conv( - "model.diffusion_model.output_blocks.7.1.proj_out", "P_bg244", "P_bg245" - ), - **easyblock( - "model.diffusion_model.output_blocks.8.0", "P_bg246", "P_bg247" - ), - **conv( - "model.diffusion_model.output_blocks.8.0.skip_connection", - "P_bg248", - "P_bg249", - ), - **norm("model.diffusion_model.output_blocks.8.1.norm", "P_bg250"), - **conv( - "model.diffusion_model.output_blocks.8.1.proj_in", "P_bg250", "P_bg251" - ), - **dense( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn1.to_q", - "P_bg252", - "P_bg253", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn1.to_k", - "P_bg252", - "P_bg253", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn1.to_v", - "P_bg252", - "P_bg253", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn1.to_out.0", - "P_bg252", - "P_bg253", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.ff.net.0.proj", - "P_bg254", - "P_bg255", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.ff.net.2", - "P_bg256", - "P_bg257", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn2.to_q", - "P_bg258", - "P_bg259", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn2.to_k", - "P_bg260", - "P_bg261", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn2.to_v", - "P_bg260", - "P_bg261", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.attn2.to_out.0", - "P_bg262", - "P_bg263", - bias=True, - ), - **norm( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.norm1", - "P_bg263", - ), - **norm( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.norm2", - "P_bg263", - ), - **norm( - "model.diffusion_model.output_blocks.8.1.transformer_blocks.0.norm3", - "P_bg263", - ), - **conv( - "model.diffusion_model.output_blocks.8.1.proj_out", "P_bg263", "P_bg264" - ), - **conv( - "model.diffusion_model.output_blocks.8.2.conv", "P_bg265", "P_bg266" - ), - **easyblock( - "model.diffusion_model.output_blocks.9.0", "P_bg267", "P_bg268" - ), - **conv( - "model.diffusion_model.output_blocks.9.0.skip_connection", - "P_bg269", - "P_bg270", - ), - **norm("model.diffusion_model.output_blocks.9.1.norm", "P_bg271"), - **conv( - "model.diffusion_model.output_blocks.9.1.proj_in", "P_bg271", "P_bg272" - ), - **dense( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn1.to_q", - "P_bg273", - "P_bg274", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn1.to_k", - "P_bg273", - "P_bg274", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn1.to_v", - "P_bg273", - "P_bg274", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn1.to_out.0", - "P_bg273", - "P_bg274", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.ff.net.0.proj", - "P_bg275", - "P_bg276", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.ff.net.2", - "P_bg277", - "P_bg278", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn2.to_q", - "P_bg279", - "P_bg280", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn2.to_k", - "P_bg281", - "P_bg282", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn2.to_v", - "P_bg281", - "P_bg282", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.attn2.to_out.0", - "P_bg283", - "P_bg284", - bias=True, - ), - **norm( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.norm1", - "P_bg284", - ), - **norm( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.norm2", - "P_bg284", - ), - **norm( - "model.diffusion_model.output_blocks.9.1.transformer_blocks.0.norm3", - "P_bg284", - ), - **conv( - "model.diffusion_model.output_blocks.9.1.proj_out", "P_bg284", "P_bg285" - ), - **easyblock( - "model.diffusion_model.output_blocks.10.0", "P_bg286", "P_bg287" - ), - **conv( - "model.diffusion_model.output_blocks.10.0.skip_connection", - "P_bg288", - "P_bg289", - ), - **norm("model.diffusion_model.output_blocks.10.1.norm", "P_bg290"), - **conv( - "model.diffusion_model.output_blocks.10.1.proj_in", "P_bg290", "P_bg291" - ), - **dense( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn1.to_q", - "P_bg292", - "P_bg293", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn1.to_k", - "P_bg292", - "P_bg293", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn1.to_v", - "P_bg292", - "P_bg293", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn1.to_out.0", - "P_bg292", - "P_bg293", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.ff.net.0.proj", - "P_b294", - "P_bg295", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.ff.net.2", - "P_bg296", - "P_bg297", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn2.to_q", - "P_bg298", - "P_bg299", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn2.to_k", - "P_bg300", - "P_bg301", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn2.to_v", - "P_bg300", - "P_bg301", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.attn2.to_out.0", - "P_bg302", - "P_bg303", - bias=True, - ), - **norm( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.norm1", - "P_bg303", - ), - **norm( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.norm2", - "P_bg303", - ), - **norm( - "model.diffusion_model.output_blocks.10.1.transformer_blocks.0.norm3", - "P_bg303", - ), - **conv( - "model.diffusion_model.output_blocks.10.1.proj_out", - "P_bg303", - "P_bg304", - ), - **easyblock( - "model.diffusion_model.output_blocks.11.0", "P_bg305", "P_bg306" - ), - **conv( - "model.diffusion_model.output_blocks.11.0.skip_connection", - "P_bg307", - "P_bg308", - ), - **norm("model.diffusion_model.output_blocks.11.1.norm", "P_bg309"), - **conv( - "model.diffusion_model.output_blocks.11.1.proj_in", "P_bg309", "P_bg310" - ), - **dense( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn1.to_q", - "P_bg311", - "P_bg312", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn1.to_k", - "P_bg311", - "P_bg312", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn1.to_v", - "P_bg311", - "P_bg312", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn1.to_out.0", - "P_bg311", - "P_bg312", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.ff.net.0.proj", - "P_bg313", - "P_bg314", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.ff.net.2", - "P_bg315", - "P_bg316", - bias=True, - ), - **dense( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn2.to_q", - "P_bg317", - "P_bg318", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn2.to_k", - "P_bg319", - "P_bg320", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn2.to_v", - "P_bg319", - "P_bg320", - bias=False, - ), - **dense( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.attn2.to_out.0", - "P_bg321", - "P_bg322", - bias=True, - ), - **norm( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.norm1", - "P_bg322", - ), - **norm( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.norm2", - "P_bg322", - ), - **norm( - "model.diffusion_model.output_blocks.11.1.transformer_blocks.0.norm3", - "P_bg322", - ), - **conv( - "model.diffusion_model.output_blocks.11.1.proj_out", - "P_bg322", - "P_bg323", - ), - **norm("model.diffusion_model.out.0", "P_bg324"), - **conv("model.diffusion_model.out.2", "P_bg325", "P_bg326"), - **skip( - "cond_stage_model.transformer.text_model.embeddings.position_ids", - None, - None, - ), - **dense( - "cond_stage_model.transformer.text_model.embeddings.token_embedding", - "P_bg365", - "P_bg366", - bias=False, - ), - **dense( - "cond_stage_model.transformer.text_model.embeddings.token_embedding", - None, - None, - ), - **dense( - "cond_stage_model.transformer.text_model.embeddings.position_embedding", - "P_bg367", - "P_bg368", - bias=False, - ), - # cond stage text encoder - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.0.self_attn.k_proj", - "P_bg369", - "P_bg370", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.0.self_attn.v_proj", - "P_bg369", - "P_bg370", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.0.self_attn.q_proj", - "P_bg369", - "P_bg370", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.0.self_attn.out_proj", - "P_bg369", - "P_bg370", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.0.layer_norm1", - "P_bg370", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.0.mlp.fc1", - "P_bg370", - "P_bg371", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.0.mlp.fc2", - "P_bg371", - "P_bg372", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.0.layer_norm2", - "P_bg372", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.1.self_attn.k_proj", - "P_bg372", - "P_bg373", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.1.self_attn.v_proj", - "P_bg372", - "P_bg373", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.1.self_attn.q_proj", - "P_bg372", - "P_bg373", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.1.self_attn.out_proj", - "P_bg372", - "P_bg373", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.1.layer_norm1", - "P_bg373", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.1.mlp.fc1", - "P_bg373", - "P_bg374", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.1.mlp.fc2", - "P_bg374", - "P_bg375", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.1.layer_norm2", - "P_bg375", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.2.self_attn.k_proj", - "P_bg375", - "P_bg376", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.2.self_attn.v_proj", - "P_bg375", - "P_bg376", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.2.self_attn.q_proj", - "P_bg375", - "P_bg376", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.2.self_attn.out_proj", - "P_bg375", - "P_bg376", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.2.layer_norm1", - "P_bg376", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.2.mlp.fc1", - "P_bg376", - "P_bg377", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.2.mlp.fc2", - "P_bg377", - "P_bg378", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.2.layer_norm2", - "P_bg378", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.3.self_attn.k_proj", - "P_bg378", - "P_bg379", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.3.self_attn.v_proj", - "P_bg378", - "P_bg379", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.3.self_attn.q_proj", - "P_bg378", - "P_bg379", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.3.self_attn.out_proj", - "P_bg378", - "P_bg379", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.3.layer_norm1", - "P_bg379", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.3.mlp.fc1", - "P_bg379", - "P_bg380", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.3.mlp.fc2", - "P_bg380", - "P_b381", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.3.layer_norm2", - "P_bg381", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.4.self_attn.k_proj", - "P_bg381", - "P_bg382", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.4.self_attn.v_proj", - "P_bg381", - "P_bg382", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.4.self_attn.q_proj", - "P_bg381", - "P_bg382", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.4.self_attn.out_proj", - "P_bg381", - "P_bg382", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.4.layer_norm1", - "P_bg382", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.4.mlp.fc1", - "P_bg382", - "P_bg383", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.4.mlp.fc2", - "P_bg383", - "P_bg384", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.4.layer_norm2", - "P_bg384", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.5.self_attn.k_proj", - "P_bg384", - "P_bg385", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.5.self_attn.v_proj", - "P_bg384", - "P_bg385", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.5.self_attn.q_proj", - "P_bg384", - "P_bg385", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.5.self_attn.out_proj", - "P_bg384", - "P_bg385", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.5.layer_norm1", - "P_bg385", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.5.mlp.fc1", - "P_bg385", - "P_bg386", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.5.mlp.fc2", - "P_bg386", - "P_bg387", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.5.layer_norm2", - "P_bg387", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.6.self_attn.k_proj", - "P_bg387", - "P_bg388", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.6.self_attn.v_proj", - "P_bg387", - "P_bg388", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.6.self_attn.q_proj", - "P_bg387", - "P_bg388", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.6.self_attn.out_proj", - "P_bg387", - "P_bg388", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.6.layer_norm1", - "P_bg389", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.6.mlp.fc1", - "P_bg389", - "P_bg390", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.6.mlp.fc2", - "P_bg390", - "P_bg391", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.6.layer_norm2", - "P_bg391", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.7.self_attn.k_proj", - "P_bg391", - "P_bg392", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.7.self_attn.v_proj", - "P_bg391", - "P_bg392", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.7.self_attn.q_proj", - "P_bg391", - "P_bg392", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.7.self_attn.out_proj", - "P_bg391", - "P_bg392", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.7.layer_norm1", - "P_bg392", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.7.mlp.fc1", - "P_bg392", - "P_bg393", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.7.mlp.fc2", - "P_bg393", - "P_bg394", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.7.layer_norm2", - "P_bg394", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.8.self_attn.k_proj", - "P_bg394", - "P_bg395", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.8.self_attn.v_proj", - "P_bg394", - "P_bg395", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.8.self_attn.q_proj", - "P_bg394", - "P_bg395", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.8.self_attn.out_proj", - "P_bg394", - "P_bg395", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.8.layer_norm1", - "P_bg395", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.8.mlp.fc1", - "P_bg395", - "P_bg396", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.8.mlp.fc2", - "P_bg396", - "P_bg397", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.8.layer_norm2", - "P_bg397", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.9.self_attn.k_proj", - "P_bg397", - "P_bg398", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.9.self_attn.v_proj", - "P_bg397", - "P_bg398", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.9.self_attn.q_proj", - "P_bg397", - "P_bg398", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.9.self_attn.out_proj", - "P_bg397", - "P_bg398", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.9.layer_norm1", - "P_bg398", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.9.mlp.fc1", - "P_bg398", - "P_bg399", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.9.mlp.fc2", - "P_bg400", - "P_bg401", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.9.layer_norm2", - "P_bg401", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.10.self_attn.k_proj", - "P_bg401", - "P_bg402", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.10.self_attn.v_proj", - "P_bg401", - "P_bg402", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.10.self_attn.q_proj", - "P_bg401", - "P_bg402", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.10.self_attn.out_proj", - "P_bg401", - "P_bg402", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.10.layer_norm1", - "P_bg402", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.10.mlp.fc1", - "P_bg402", - "P_bg403", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.10.mlp.fc2", - "P_bg403", - "P_bg404", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.10.layer_norm2", - "P_bg404", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.11.self_attn.k_proj", - "P_bg404", - "P_bg405", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.11.self_attn.v_proj", - "P_bg404", - "P_bg405", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.11.self_attn.q_proj", - "P_bg404", - "P_bg405", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.11.self_attn.out_proj", - "P_bg404", - "P_bg405", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.11.layer_norm1", - "P_bg405", - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.11.mlp.fc1", - "P_bg405", - "P_bg406", - bias=True, - ), - **dense( - "cond_stage_model.transformer.text_model.encoder.layers.11.mlp.fc2", - "P_bg406", - "P_bg407", - bias=True, - ), - **norm( - "cond_stage_model.transformer.text_model.encoder.layers.11.layer_norm2", - "P_bg407", - ), - **norm( - "cond_stage_model.transformer.text_model.final_layer_norm", "P_bg407" - ), - } - ) - - def get_permuted_param(ps: PermutationSpec, perm, k: str, params, except_axis=None): """Get parameter `k` from `params`, with the permutations applied.""" w = params[k] diff --git a/modules/modelloader.py b/modules/modelloader.py index 4195e6905..0e8254024 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -205,7 +205,7 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config download_config["mirror"] = mirror if custom_pipeline is not None and len(custom_pipeline) > 0: download_config["custom_pipeline"] = custom_pipeline - shared.log.debug(f"Diffusers downloading: {hub_id} {download_config}") + shared.log.debug(f"Diffusers downloading: {hub_id} args={download_config}") if token is not None and len(token) > 2: shared.log.debug(f"Diffusers authentication: {token}") hf.login(token) @@ -574,6 +574,7 @@ def move_files(src_path: str, dest_path: str, ext_filter: str = None): def load_upscalers(): # We can only do this 'magic' method to dynamically load upscalers if they are referenced, so we'll try to import any _model.py files before looking in __subclasses__ + t0 = time.time() modules_dir = os.path.join(shared.script_path, "modules", "postprocess") for file in os.listdir(modules_dir): if "_model.py" in file: @@ -602,4 +603,5 @@ def load_upscalers(): datas += scaler.scalers names.append(name[8:]) shared.sd_upscalers = sorted(datas, key=lambda x: x.name.lower() if not isinstance(x.scaler, (UpscalerNone, UpscalerLanczos, UpscalerNearest)) else "") # Special case for UpscalerNone keeps it at the beginning of the list. - shared.log.debug(f"Load upscalers: total={len(shared.sd_upscalers)} downloaded={len([x for x in shared.sd_upscalers if x.data_path is not None and os.path.isfile(x.data_path)])} user={len([x for x in shared.sd_upscalers if x.custom])} {names}") + t1 = time.time() + shared.log.debug(f"Load upscalers: total={len(shared.sd_upscalers)} downloaded={len([x for x in shared.sd_upscalers if x.data_path is not None and os.path.isfile(x.data_path)])} user={len([x for x in shared.sd_upscalers if x.custom])} time={t1-t0:.2f} {names}") diff --git a/modules/processing.py b/modules/processing.py index 3c3faada5..b04388d8f 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -18,7 +18,7 @@ from ldm.models.diffusion.ddpm import LatentDepth2ImageDiffusion from einops import repeat, rearrange from blendmodes.blend import blendLayers, BlendType from installer import git_commit -from modules import shared, devices +from modules import shared, devices, errors import modules.memstats import modules.lowvram import modules.masking @@ -121,8 +121,7 @@ class StableDiffusionProcessing: """ The first set of paramaters: sd_models -> do_not_reload_embeddings represent the minimum required to create a StableDiffusionProcessing """ - def __init__(self, sd_model=None, outpath_samples=None, outpath_grids=None, prompt: str = "", styles: List[str] = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, seed_enable_extras: bool = True, sampler_name: str = None, latent_sampler: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, cfg_scale: float = 7.0, image_cfg_scale: float = None, clip_skip: int = 1, width: int = 512, height: int = 512, full_quality: bool = True, restore_faces: bool = False, tiling: bool = False, do_not_save_samples: bool = False, do_not_save_grid: bool = False, extra_generation_params: Dict[Any, Any] = None, overlay_images: Any = None, negative_prompt: str = None, eta: float = None, do_not_reload_embeddings: bool = False, denoising_strength: float = 0, diffusers_guidance_rescale: float = 0.7, override_settings: Dict[str, Any] = None, override_settings_restore_afterwards: bool = True, sampler_index: int = None, script_args: list = None): # pylint: disable=unused-argument - + def __init__(self, sd_model=None, outpath_samples=None, outpath_grids=None, prompt: str = "", styles: List[str] = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, seed_enable_extras: bool = True, sampler_name: str = None, latent_sampler: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, cfg_scale: float = 7.0, image_cfg_scale: float = None, clip_skip: int = 1, width: int = 512, height: int = 512, full_quality: bool = True, restore_faces: bool = False, tiling: bool = False, do_not_save_samples: bool = False, do_not_save_grid: bool = False, extra_generation_params: Dict[Any, Any] = None, overlay_images: Any = None, negative_prompt: str = None, eta: float = None, do_not_reload_embeddings: bool = False, denoising_strength: float = 0, diffusers_guidance_rescale: float = 0.7, hdr_clamp: bool = False, hdr_boundary: float = 4.0, hdr_threshold: float = 3.5, hdr_center: bool = False, hdr_channel_shift: float = 0.8, hdr_full_shift: float = 0.8, hdr_maximize: bool = False, hdr_max_center: float = 0.6, hdr_max_boundry: float = 1.0, override_settings: Dict[str, Any] = None, override_settings_restore_afterwards: bool = True, sampler_index: int = None, script_args: list = None): # pylint: disable=unused-argument self.outpath_samples: str = outpath_samples self.outpath_grids: str = outpath_grids self.prompt: str = prompt @@ -166,8 +165,8 @@ class StableDiffusionProcessing: self.disable_extra_networks = False self.token_merging_ratio = 0 self.token_merging_ratio_hr = 0 - self.scripts = None - self.script_args = script_args or [] + # self.scripts = modules.scripts.ScriptRunner() # set via property + # self.script_args = script_args or [] # set via property self.per_script_args = {} self.all_prompts = None self.all_negative_prompts = None @@ -192,6 +191,7 @@ class StableDiffusionProcessing: self.s_tmin = shared.opts.s_tmin self.s_tmax = float('inf') # not representable as a standard ui option shared.opts.data['clip_skip'] = clip_skip + self.task_args = {} # TODO a1111 compatibility items self.refiner_switch_at = 0 self.hr_prompt = '' @@ -203,6 +203,16 @@ class StableDiffusionProcessing: self.scripts_value: modules.scripts.ScriptRunner = field(default=None, init=False) self.script_args_value: list = field(default=None, init=False) self.scripts_setup_complete: bool = field(default=False, init=False) + # hdr + self.hdr_clamp = hdr_clamp + self.hdr_boundary = hdr_boundary + self.hdr_threshold = hdr_threshold + self.hdr_center = hdr_center + self.hdr_channel_shift = hdr_channel_shift + self.hdr_full_shift = hdr_full_shift + self.hdr_maximize = hdr_maximize + self.hdr_max_center = hdr_max_center + self.hdr_max_boundry = hdr_max_boundry @property @@ -533,12 +543,20 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No index = position_in_batch + iteration * p.batch_size if all_prompts is None: all_prompts = p.all_prompts + if all_negative_prompts is None: + all_negative_prompts = p.all_negative_prompts if all_seeds is None: all_seeds = p.all_seeds if all_subseeds is None: all_subseeds = p.all_subseeds - if all_negative_prompts is None: - all_negative_prompts = p.all_negative_prompts + while len(all_prompts) <= index: + all_prompts.append(all_prompts[-1]) + while len(all_seeds) <= index: + all_seeds.append(all_seeds[-1]) + while len(all_subseeds) <= index: + all_subseeds.append(all_subseeds[-1]) + while len(all_negative_prompts) <= index: + all_negative_prompts.append(all_negative_prompts[-1]) comment = ', '.join(comments) if comments is not None and type(comments) is list else None ops = list(set(p.ops)) ops.reverse() @@ -597,10 +615,14 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No args["Init image size"] = f"{getattr(p, 'init_img_width', 0)}x{getattr(p, 'init_img_height', 0)}" args["Init image hash"] = getattr(p, 'init_img_hash', None) args["Mask weight"] = getattr(p, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) if p.is_using_inpainting_conditioning else None - args['Resize mode'] = getattr(p, 'resize_mode', None) args['Resize scale'] = getattr(p, 'scale_by', None) args["Mask blur"] = p.mask_blur if getattr(p, 'mask', None) is not None and getattr(p, 'mask_blur', 0) > 0 else None args["Denoising strength"] = getattr(p, 'denoising_strength', None) + # lookup by index + if getattr(p, 'resize_mode', None) is not None: + RESIZE_MODES = ["None", "Resize fixed", "Crop and resize", "Resize and fill", "Latent upscale"] + args['Resize mode'] = RESIZE_MODES[p.resize_mode] + # TODO missing-by-index: inpainting_fill, inpaint_full_res, inpainting_mask_invert if 'face' in p.ops: args["Face restoration"] = shared.opts.face_restoration_model if 'color' in p.ops: @@ -644,36 +666,6 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No return infotext -""" -def print_profile(profile, msg: str): - try: - from rich import print # pylint: disable=redefined-builtin - except Exception: - pass - lines = profile.key_averages().table(sort_by="cuda_time_total", row_limit=20) - lines = lines.split('\n') - lines = [l for l in lines if '/profiler' not in l] - print(f'Profile {msg}:', '\n'.join(lines)) -""" - - -def print_profile(profile, msg: str): - import io - import pstats - try: - from rich import print # pylint: disable=redefined-builtin - except Exception: - pass - profile.disable() - stream = io.StringIO() # pylint: disable=abstract-class-instantiated - ps = pstats.Stats(profile, stream=stream) - ps.sort_stats(pstats.SortKey.CUMULATIVE).print_stats(15) - profile = None - lines = stream.getvalue().split('\n') - lines = [line for line in lines if ' Processed: if not hasattr(p.sd_model, 'sd_checkpoint_info'): return None @@ -721,22 +713,26 @@ def process_images(p: StableDiffusionProcessing) -> Processed: modules.script_callbacks.before_process_callback(p) if shared.cmd_opts.profile: - """ - import torch.profiler # pylint: disable=redefined-outer-name - with torch.profiler.profile(profile_memory=True, with_modules=True) as prof: - with torch.profiler.record_function("process_images"): - res = process_images_inner(p) - print_profile(prof, 'process_images') - """ import cProfile - pr = cProfile.Profile() - pr.enable() + profile_python = cProfile.Profile() + profile_python.enable() with context_hypertile_vae(p), context_hypertile_unet(p): + import torch.profiler # pylint: disable=redefined-outer-name + activities=[torch.profiler.ProfilerActivity.CPU] + if torch.cuda.is_available(): + activities.append(torch.profiler.ProfilerActivity.CUDA) + shared.log.debug(f'Torch profile: activities={activities}') + if shared.profiler is None: + shared.profiler = torch.profiler.profile(activities=activities, profile_memory=True, with_modules=True) + shared.profiler.start() + shared.profiler.step() res = process_images_inner(p) - print_profile(pr, 'Torch') + errors.profile_torch(shared.profiler, 'Process') + errors.profile(profile_python, 'Process') else: with context_hypertile_vae(p), context_hypertile_unet(p): res = process_images_inner(p) + finally: if not shared.opts.cuda_compile: modules.sd_models.apply_token_merging(p.sd_model, 0) @@ -754,12 +750,14 @@ def process_images(p: StableDiffusionProcessing) -> Processed: def validate_sample(tensor): + if not isinstance(tensor, np.ndarray) and not isinstance(tensor, torch.Tensor): + return tensor if tensor.dtype == torch.bfloat16: # numpy does not support bf16 tensor = tensor.to(torch.float16) if shared.backend == shared.Backend.ORIGINAL: sample = 255.0 * np.moveaxis(tensor.cpu().numpy(), 0, 2) else: - sample = 255. * tensor + sample = 255.0 * tensor with warnings.catch_warnings(record=True) as w: cast = sample.astype(np.uint8) if len(w) > 0: @@ -1045,6 +1043,8 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.refiner_prompt = refiner_prompt self.refiner_negative = refiner_negative self.sampler = None + self.scripts = None + self.script_args = [] def init(self, all_prompts, all_seeds, all_subseeds): if shared.backend == shared.Backend.DIFFUSERS: @@ -1207,6 +1207,8 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): self.is_batch = False self.scale_by = 1.0 self.sampler = None + self.scripts = None + self.script_args = [] def init(self, all_prompts, all_seeds, all_subseeds): if shared.backend == shared.Backend.DIFFUSERS and self.image_mask is not None: diff --git a/modules/processing_correction.py b/modules/processing_correction.py new file mode 100644 index 000000000..704805db4 --- /dev/null +++ b/modules/processing_correction.py @@ -0,0 +1,63 @@ +""" +based on article by TimothyAlexisVass +https://huggingface.co/blog/TimothyAlexisVass/explaining-the-sdxl-latent-space +""" + +import os +import torch +from modules import shared + + +debug = shared.log.info if os.environ.get('SD_HDR_DEBUG', None) is not None else lambda *args, **kwargs: None + + +def soft_clamp_tensor(input_tensor, threshold=0.8, boundary=4): + # shrinking towards the mean; will also remove outliers + if max(abs(input_tensor.max()), abs(input_tensor.min())) < boundary or threshold == 0: + return input_tensor + channel_dim = 1 + threshold *= boundary + max_vals = input_tensor.max(channel_dim, keepdim=True)[0] + max_replace = ((input_tensor - threshold) / (max_vals - threshold)) * (boundary - threshold) + threshold + over_mask = input_tensor > threshold + min_vals = input_tensor.min(channel_dim, keepdim=True)[0] + min_replace = ((input_tensor + threshold) / (min_vals + threshold)) * (-boundary + threshold) - threshold + under_mask = input_tensor < -threshold + debug(f'HDE soft clamp: threshold={threshold} boundary={boundary}') + input_tensor = torch.where(over_mask, max_replace, torch.where(under_mask, min_replace, input_tensor)) + return input_tensor + + +def center_tensor(input_tensor, channel_shift=1.0, full_shift=1.0, channels=[0, 1, 2, 3]): # pylint: disable=dangerous-default-value # noqa: B006 + if channel_shift == 0 and full_shift == 0: + return input_tensor + means = [] + for channel in channels: + means.append(input_tensor[0, channel].mean()) + input_tensor[0, channel] -= means[-1] * channel_shift + debug(f'HDR center: channel-shift{channel_shift} full-shift={full_shift} means={torch.stack(means)}') + input_tensor = input_tensor - input_tensor.mean() * full_shift + return input_tensor + + +def maximize_tensor(input_tensor, boundary=1.0, channels=[0, 1, 2]): # pylint: disable=dangerous-default-value # noqa: B006 + if boundary == 1.0: + return input_tensor + boundary *= 4 + min_val = input_tensor.min() + max_val = input_tensor.max() + normalization_factor = boundary / max(abs(min_val), abs(max_val)) + input_tensor[0, channels] *= normalization_factor + debug(f'HDR maximize: boundary={boundary} min={min_val} max={max_val} factor={normalization_factor}') + return input_tensor + + +def correction_callback(p, timestep, kwags): + if timestep > 950 and p.hdr_clamp: + kwags["latents"] = soft_clamp_tensor(kwags["latents"], threshold=p.hdr_threshold, boundary=p.hdr_boundary) + if timestep > 700 and p.hdr_center: + kwags["latents"] = center_tensor(kwags["latents"], channel_shift=p.hdr_channel_shift, full_shift=p.hdr_full_shift) + if timestep > 1 and timestep < 100 and p.hdr_maximize: + kwags["latents"] = center_tensor(kwags["latents"], channel_shift=p.hdr_max_center, full_shift=1.0) + kwags["latents"] = maximize_tensor(kwags["latents"], boundary=p.hdr_max_boundry) + return kwags diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 28f1a553d..e90cd0126 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -5,7 +5,6 @@ import inspect import typing import torch import torchvision.transforms.functional as TF -import diffusers import modules.devices as devices import modules.shared as shared import modules.sd_samplers as sd_samplers @@ -17,6 +16,7 @@ import modules.errors as errors from modules.processing import StableDiffusionProcessing, create_random_tensors import modules.prompt_parser_diffusers as prompt_parser_diffusers from modules.sd_hijack_hypertile import hypertile_set +from modules.processing_correction import correction_callback def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_prompts): @@ -71,10 +71,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro raise AssertionError('Interrupted...') time.sleep(0.1) - def diffusers_callback(_pipe, step: int, _timestep: int, kwargs: dict): - latents = kwargs['latents'] + def diffusers_callback(_pipe, step: int, timestep: int, kwargs: dict): shared.state.sampling_step = step - shared.state.current_latent = latents if shared.state.interrupted or shared.state.skipped: raise AssertionError('Interrupted...') if shared.state.paused: @@ -83,7 +81,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if shared.state.interrupted or shared.state.skipped: raise AssertionError('Interrupted...') time.sleep(0.1) - return {'latents': latents} + if kwargs.get('latents', None) is None: + return kwargs + kwargs = correction_callback(p, timestep, kwargs) + shared.state.current_latent = kwargs['latents'] + if shared.cmd_opts.profile and shared.profiler is not None: + shared.profiler.step() + return kwargs def full_vae_decode(latents, model): t0 = time.time() @@ -127,7 +131,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if len(latents) == 0: return [] decoded = torch.zeros((len(latents), 3, latents.shape[2] * 8, latents.shape[3] * 8), dtype=devices.dtype_vae, device=devices.device) - for i in range(len(output.images)): + for i in range(latents.shape[0]): decoded[i] = sd_vae_taesd.decode(latents[i]) return decoded @@ -137,6 +141,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro return encoded def vae_decode(latents, model, output_type='np', full_quality=True): + t0 = time.time() prev_job = shared.state.job shared.state.job = 'vae' if not torch.is_tensor(latents): # already decoded @@ -149,6 +154,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if not hasattr(model, 'vae'): shared.log.error('VAE not found in model') return [] + if latents.shape[0] == 4 and latents.shape[1] != 4: # likely animatediff latent + latents = latents.permute(1, 0, 2, 3) if len(latents.shape) == 3: # lost a batch dim in hires latents = latents.unsqueeze(0) if full_quality: @@ -159,6 +166,9 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro # decoded = validate_sample(decoded) imgs = model.image_processor.postprocess(decoded, output_type=output_type) shared.state.job = prev_job + if shared.cmd_opts.profile: + t1 = time.time() + shared.log.debug(f'Profile: VAE decode: {t1-t0:.2f}') return imgs def vae_encode(image, model, full_quality=True): # pylint: disable=unused-variable @@ -198,16 +208,27 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro def task_specific_kwargs(model): task_args = {} - is_img2img_model = bool("Zero123" in shared.sd_model.__class__.__name__) + is_img2img_model = bool('Zero123' in shared.sd_model.__class__.__name__) if sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE and not is_img2img_model: p.ops.append('txt2img') - task_args = {"height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8)} + task_args = { + 'height': 8 * math.ceil(p.height / 8), + 'width': 8 * math.ceil(p.width / 8), + } elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or is_img2img_model) and len(getattr(p, 'init_images' ,[])) > 0: p.ops.append('img2img') - task_args = {"image": p.init_images, "strength": p.denoising_strength} + task_args = { + 'image': p.init_images, + 'strength': p.denoising_strength, + } elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INSTRUCT and len(getattr(p, 'init_images' ,[])) > 0: p.ops.append('instruct') - task_args = {"height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8), "image": p.init_images, "strength": p.denoising_strength} + task_args = { + 'height': 8 * math.ceil(p.height / 8), + 'width': 8 * math.ceil(p.width / 8), + 'image': p.init_images, + 'strength': p.denoising_strength, + } elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INPAINTING or is_img2img_model) and len(getattr(p, 'init_images' ,[])) > 0: p.ops.append('inpaint') if getattr(p, 'mask', None) is None: @@ -215,7 +236,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro width = 8 * math.ceil(p.init_images[0].width / 8) height = 8 * math.ceil(p.init_images[0].height / 8) # option-1: use images as inputs - task_args = {"image": p.init_images, "mask_image": p.mask, "strength": p.denoising_strength, "height": height, "width": width} + task_args = { + 'image': p.init_images, + 'mask_image': p.mask, + 'strength': p.denoising_strength, + 'height': height, + 'width': width, + } """ # option-2: preprocess images into latents using diffusers vae_scale_factor = 2 ** (len(model.vae.config.block_out_channels) - 1) image_processor = diffusers.image_processor.VaeImageProcessor(vae_scale_factor=vae_scale_factor) @@ -235,14 +262,20 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro task_args = {"image": p.init_images, "mask_image": mask_image, "masked_image_latents": masked_image_latents, "strength": p.denoising_strength, "height": height, "width": width} """ if model.__class__.__name__ == 'LatentConsistencyModelPipeline' and hasattr(p, 'init_images') and len(p.init_images) > 0: + p.ops.append('lcm') init_latents = [vae_encode(image, model=shared.sd_model, full_quality=p.full_quality).squeeze(dim=0) for image in p.init_images] init_latent = torch.stack(init_latents, dim=0).to(shared.device) init_noise = p.denoising_strength * create_random_tensors(init_latent.shape[1:], seeds=p.all_seeds, subseeds=p.all_subseeds, subseed_strength=p.subseed_strength, p=p) init_latent = (1 - p.denoising_strength) * init_latent + init_noise - task_args = {"latents": init_latent.to(model.dtype), "width": p.width, "height": p.height } + task_args = { + 'latents': init_latent.to(model.dtype), + 'width': p.width, + 'height': p.height, + } return task_args def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs): + t0 = time.time() if hasattr(model, "set_progress_bar_config"): model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba') args = {} @@ -307,12 +340,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro pass task_kwargs = task_specific_kwargs(model) for arg in task_kwargs: - if arg in possible and arg not in args: # task specific args should not override args + # if arg in possible and arg not in args: # task specific args should not override args + if arg in possible: args[arg] = task_kwargs[arg] - else: - pass - # shared.log.debug(f'Diffuser not supported: pipeline={pipeline.__class__.__name__} task={sd_models.get_diffusers_task(model)} arg={arg}') - # shared.log.debug(f'Diffuser pipeline: {model.__class__.__name__} possible={possible}') + task_args = getattr(p, 'task_args', {}) + for k, v in task_args.items(): + args[k] = v + hypertile_set(p, hr=len(getattr(p, 'init_images', []))) clean = args.copy() clean.pop('callback', None) @@ -327,6 +361,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro clean['mask_image'] = type(clean['mask_image']) if 'masked_image_latents' in clean: clean['masked_image_latents'] = type(clean['masked_image_latents']) + if 'ip_adapter_image' in clean: + clean['ip_adapter_image'] = type(clean['ip_adapter_image']) if 'prompt' in clean: clean['prompt'] = len(clean['prompt']) if 'negative_prompt' in clean: @@ -342,8 +378,17 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro clean['generator'] = generator_device clean['parser'] = parser shared.log.debug(f'Diffuser pipeline: {model.__class__.__name__} task={sd_models.get_diffusers_task(model)} set={clean}') + if p.hdr_clamp or p.hdr_center or p.hdr_maximize: + txt = 'HDR:' + txt += f' Clamp threshold={p.hdr_threshold} boundary={p.hdr_boundary}' if p.hdr_clamp else 'Clamp off' + txt += f' Center channel-shift={p.hdr_channel_shift} full-shift={p.hdr_full_shift}' if p.hdr_center else 'Center off' + txt += f' Maximize boundary={p.hdr_max_boundry} center={p.hdr_max_center}' if p.hdr_maximize else 'Maximize off' + shared.log.debug(txt) # components = [{ k: getattr(v, 'device', None) } for k, v in model.components.items()] # shared.log.debug(f'Diffuser pipeline components: {components}') + if shared.cmd_opts.profile: + t1 = time.time() + shared.log.debug(f'Profile: pipeline args: {t1-t0:.2f}') return args def recompile_model(hires=False): @@ -375,8 +420,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro def update_sampler(sd_model, second_pass=False): sampler_selection = p.latent_sampler if second_pass else p.sampler_name - is_karras_compatible = sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers - if hasattr(sd_model, 'scheduler') and sampler_selection != 'Default' and is_karras_compatible: + # is_karras_compatible = sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers + if hasattr(sd_model, 'scheduler') and sampler_selection != 'Default': sampler = sd_samplers.all_samplers_map.get(sampler_selection, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") @@ -384,10 +429,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro # TODO extra_generation_params add sampler options # p.extra_generation_params['Sampler options'] = '' - recompile_model() - update_sampler(shared.sd_model) - p.extra_generation_params['Pipeline'] = shared.sd_model.__class__.__name__ - if len(getattr(p, 'init_images', [])) > 0: while len(p.init_images) < len(prompts): p.init_images.append(p.init_images[-1]) @@ -457,6 +498,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts, num_inference_steps=calculate_base_steps(), eta=shared.opts.scheduler_eta, + guidance_scale=p.cfg_scale, guidance_rescale=p.diffusers_guidance_rescale, denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None, denoising_end=p.refiner_start if use_refiner_start else 1 if use_denoise_start else None, @@ -464,17 +506,31 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro clip_skip=p.clip_skip, desc='Base', ) + recompile_model() + update_sampler(shared.sd_model) shared.state.sampling_steps = base_args['num_inference_steps'] + p.extra_generation_params['Pipeline'] = shared.sd_model.__class__.__name__ p.extra_generation_params["Sampler Eta"] = shared.opts.scheduler_eta if shared.opts.scheduler_eta is not None and shared.opts.scheduler_eta > 0 and shared.opts.scheduler_eta < 1 else None try: + t0 = time.time() output = shared.sd_model(**base_args) # pylint: disable=not-callable + if shared.cmd_opts.profile: + t1 = time.time() + shared.log.debug(f'Profile: pipeline call: {t1-t0:.2f}') + if not hasattr(output, 'images') and hasattr(output, 'frames'): + shared.log.debug(f'Generated: frames={len(output.frames[0])}') + output.images = output.frames[0] except AssertionError as e: shared.log.info(e) except ValueError as e: shared.state.interrupted = True - shared.log.error(f'Processing: {e}') + shared.log.error(f'Processing: args={base_args} {e}') if shared.cmd_opts.debug: errors.display(e, 'Processing') + except RuntimeError as e: + shared.state.interrupted = True + shared.log.error(f'Processing: args={base_args} {e}') + errors.display(e, 'Processing') if hasattr(shared.sd_model, 'embedding_db') and len(shared.sd_model.embedding_db.embeddings_used) > 0: p.extra_generation_params['Embeddings'] = ', '.join(shared.sd_model.embedding_db.embeddings_used) diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 735cf995a..abd06b778 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -59,9 +59,9 @@ class DiffusersTextualInversionManager(BaseTextualInversionManager): return self.pipe.tokenizer.encode(prompt, add_special_tokens=False) -def encode_prompts(pipeline, prompts: list, negative_prompts: list, clip_skip: typing.Optional[int] = None): - if 'StableDiffusion' not in pipeline.__class__.__name__: - shared.log.warning(f"Prompt parser not supported: {pipeline.__class__.__name__}") +def encode_prompts(pipe, prompts: list, negative_prompts: list, clip_skip: typing.Optional[int] = None): + if 'StableDiffusion' not in pipe.__class__.__name__: + shared.log.warning(f"Prompt parser not supported: {pipe.__class__.__name__}") return None, None, None, None else: prompt_embeds = [] @@ -69,7 +69,7 @@ def encode_prompts(pipeline, prompts: list, negative_prompts: list, clip_skip: t negative_embeds = [] negative_pooleds = [] for i in range(len(prompts)): - prompt_embed, positive_pooled, negative_embed, negative_pooled = get_weighted_text_embeddings(pipeline, prompts[i], negative_prompts[i], clip_skip) + prompt_embed, positive_pooled, negative_embed, negative_pooled = get_weighted_text_embeddings(pipe, prompts[i], negative_prompts[i], clip_skip) prompt_embeds.append(prompt_embed) positive_pooleds.append(positive_pooled) negative_embeds.append(negative_embed) @@ -118,9 +118,9 @@ def prepare_embedding_providers(pipe, clip_skip): def pad_to_same_length(pipe, embeds): device = pipe.device if str(pipe.device) != 'meta' else devices.device try: #SDXL - empty_embed = shared.sd_model.encode_prompt("") + empty_embed = pipe.encode_prompt("") except Exception: #SD1.5 - empty_embed = shared.sd_model.encode_prompt("", device, 1, False) + empty_embed = pipe.encode_prompt("", device, 1, False) empty_batched = torch.cat([empty_embed[0].to(embeds[0].device)] * embeds[0].shape[0]) max_token_count = max([embed.shape[1] for embed in embeds]) for i, embed in enumerate(embeds): diff --git a/modules/rife/__init__.py b/modules/rife/__init__.py new file mode 100644 index 000000000..a7db7d9a2 --- /dev/null +++ b/modules/rife/__init__.py @@ -0,0 +1,117 @@ +#!/bin/env python + +import _thread +import os +import time +from queue import Queue +import cv2 +import numpy as np +import torch +from PIL import Image +from torch.nn import functional as F +from tqdm.rich import tqdm +from modules.rife.ssim import ssim_matlab +from modules.rife.model_rife import Model +from modules import devices, shared + + +model_url = 'https://github.com/vladmandic/rife/raw/main/model/flownet-v46.pkl' +model = None + + +def load(model_path: str = 'rife/flownet-v46.pkl'): + global model # pylint: disable=global-statement + if model is None: + from modules import modelloader + model_dir = os.path.join(shared.models_path, 'RIFE') + model_path = modelloader.load_file_from_url(url=model_url, model_dir=model_dir, file_name='flownet-v46.pkl') + shared.log.debug(f'RIFE load model: file="{model_path}"') + model = Model() + model.load_model(model_path, -1) + model.eval() + model.device() + + +def interpolate(images: list, count: int = 2, scale: float = 1.0, pad: int = 1, change: float = 0.3): + if images is None or len(images) < 2: + return [] + if model is None: + load() + interpolated = [] + h = images[0].height + w = images[0].width + t0 = time.time() + + def write(buffer): + item = buffer.get() + while item is not None: + img = item[:, :, ::-1] + # image = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) + image = Image.fromarray(img) + item = buffer.get() + interpolated.append(image) + + def execute(I0, I1, n): + if model.version >= 3.9: + res = [] + for i in range(n): + res.append(model.inference(I0, I1, (i+1) * 1. / (n+1), scale)) + return res + else: + middle = model.inference(I0, I1, scale) + if n == 1: + return [middle] + first_half = execute(I0, middle, n=n//2) + second_half = execute(middle, I1, n=n//2) + if n % 2: + return [*first_half, middle, *second_half] + else: + return [*first_half, *second_half] + + def f_pad(img): + return F.pad(img, padding).to(devices.dtype) # pylint: disable=not-callable + + tmp = max(128, int(128 / scale)) + ph = ((h - 1) // tmp + 1) * tmp + pw = ((w - 1) // tmp + 1) * tmp + padding = (0, pw - w, 0, ph - h) + buffer = Queue(maxsize=8192) + _thread.start_new_thread(write, (buffer,)) + + frame = cv2.cvtColor(np.array(images[0]), cv2.COLOR_RGB2BGR) + for _i in range(pad): # fill starting frames + buffer.put(frame) + + I1 = f_pad(torch.from_numpy(np.transpose(frame, (2,0,1))).to(devices.device, non_blocking=True).unsqueeze(0).float() / 255.) + with torch.no_grad(): + with tqdm(total=len(images), desc='Interpolate', unit='frame') as pbar: + for image in images: + frame = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR) + I0 = I1 + I1 = f_pad(torch.from_numpy(np.transpose(frame, (2,0,1))).to(devices.device, non_blocking=True).unsqueeze(0).float() / 255.) + I0_small = F.interpolate(I0, (32, 32), mode='bilinear', align_corners=False).to(torch.float32) + I1_small = F.interpolate(I1, (32, 32), mode='bilinear', align_corners=False).to(torch.float32) + ssim = ssim_matlab(I0_small[:, :3], I1_small[:, :3]) + if ssim > 0.99: # skip duplicate frames + continue + if ssim < change: + output = [] + for _i in range(pad): # fill frames if change rate is above threshold + output.append(I0) + for _i in range(pad): + output.append(I1) + else: + output = execute(I0, I1, count-1) + for mid in output: + mid = (((mid[0] * 255.).byte().cpu().numpy().transpose(1, 2, 0))) + buffer.put(mid[:h, :w]) + buffer.put(frame) + pbar.update(1) + + for _i in range(pad): # fill ending frames + buffer.put(frame) + while not buffer.empty(): + time.sleep(0.1) + t1 = time.time() + shared.log.info(f'RIFE interpolate: input={len(images)} frames={len(interpolated)} resolution={w}x{h} interpolate={count} scale={scale} pad={pad} change={change} time={round(t1 - t0, 2)}') + return interpolated diff --git a/modules/rife/loss.py b/modules/rife/loss.py new file mode 100644 index 000000000..993f319fb --- /dev/null +++ b/modules/rife/loss.py @@ -0,0 +1,122 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision.models as models +from modules import devices + + +class EPE(nn.Module): + def __init__(self): + super(EPE, self).__init__() + + def forward(self, flow, gt, loss_mask): + loss_map = (flow - gt.detach()) ** 2 + loss_map = (loss_map.sum(1, True) + 1e-6) ** 0.5 + return loss_map * loss_mask + + +class Ternary(nn.Module): + def __init__(self): + super(Ternary, self).__init__() + patch_size = 7 + out_channels = patch_size * patch_size + self.w = np.eye(out_channels).reshape( + (patch_size, patch_size, 1, out_channels)) + self.w = np.transpose(self.w, (3, 2, 0, 1)) + self.w = torch.tensor(self.w).float().to(devices.device) + + def transform(self, img): + patches = F.conv2d(img, self.w, padding=3, bias=None) + transf = patches - img + transf_norm = transf / torch.sqrt(0.81 + transf**2) + return transf_norm + + def rgb2gray(self, rgb): + r, g, b = rgb[:, 0:1, :, :], rgb[:, 1:2, :, :], rgb[:, 2:3, :, :] + gray = 0.2989 * r + 0.5870 * g + 0.1140 * b + return gray + + def hamming(self, t1, t2): + dist = (t1 - t2) ** 2 + dist_norm = torch.mean(dist / (0.1 + dist), 1, True) + return dist_norm + + def valid_mask(self, t, padding): + n, _, h, w = t.size() + inner = torch.ones(n, 1, h - 2 * padding, w - 2 * padding).type_as(t) + mask = F.pad(inner, [padding] * 4) + return mask + + def forward(self, img0, img1): + img0 = self.transform(self.rgb2gray(img0)) + img1 = self.transform(self.rgb2gray(img1)) + return self.hamming(img0, img1) * self.valid_mask(img0, 1) + + +class SOBEL(nn.Module): + def __init__(self): + super(SOBEL, self).__init__() + self.kernelX = torch.tensor([ + [1, 0, -1], + [2, 0, -2], + [1, 0, -1], + ]).float() + self.kernelY = self.kernelX.clone().T + self.kernelX = self.kernelX.unsqueeze(0).unsqueeze(0).to(devices.device) + self.kernelY = self.kernelY.unsqueeze(0).unsqueeze(0).to(devices.device) + + def forward(self, pred, gt): + N, C, H, W = pred.shape[0], pred.shape[1], pred.shape[2], pred.shape[3] + img_stack = torch.cat( + [pred.reshape(N*C, 1, H, W), gt.reshape(N*C, 1, H, W)], 0) + sobel_stack_x = F.conv2d(img_stack, self.kernelX, padding=1) + sobel_stack_y = F.conv2d(img_stack, self.kernelY, padding=1) + pred_X, gt_X = sobel_stack_x[:N*C], sobel_stack_x[N*C:] + pred_Y, gt_Y = sobel_stack_y[:N*C], sobel_stack_y[N*C:] + L1X, L1Y = torch.abs(pred_X-gt_X), torch.abs(pred_Y-gt_Y) + loss = L1X+L1Y + return loss + + +class MeanShift(nn.Conv2d): + def __init__(self, data_mean, data_std, data_range=1, norm=True): + c = len(data_mean) + super(MeanShift, self).__init__(c, c, kernel_size=1) + std = torch.Tensor(data_std) + self.weight.data = torch.eye(c).view(c, c, 1, 1) + if norm: + self.weight.data.div_(std.view(c, 1, 1, 1)) + self.bias.data = -1 * data_range * torch.Tensor(data_mean) + self.bias.data.div_(std) + else: + self.weight.data.mul_(std.view(c, 1, 1, 1)) + self.bias.data = data_range * torch.Tensor(data_mean) + self.requires_grad = False + + +class VGGPerceptualLoss(torch.nn.Module): + def __init__(self, rank=0): # pylint: disable=unused-argument + super(VGGPerceptualLoss, self).__init__() + pretrained = True + self.vgg_pretrained_features = models.vgg19( + pretrained=pretrained).features + self.normalize = MeanShift([0.485, 0.456, 0.406], [ + 0.229, 0.224, 0.225], norm=True).cuda() + for param in self.parameters(): + param.requires_grad = False + + def forward(self, X, Y, indices=None): + X = self.normalize(X) + Y = self.normalize(Y) + indices = [2, 7, 12, 21, 30] + weights = [1.0/2.6, 1.0/4.8, 1.0/3.7, 1.0/5.6, 10/1.5] + k = 0 + loss = 0 + for i in range(indices[-1]): + X = self.vgg_pretrained_features[i](X) + Y = self.vgg_pretrained_features[i](Y) + if i+1 in indices: + loss += weights[k] * (X - Y.detach()).abs().mean() * 0.1 + k += 1 + return loss diff --git a/modules/rife/model_ifnet.py b/modules/rife/model_ifnet.py new file mode 100644 index 000000000..ec667425b --- /dev/null +++ b/modules/rife/model_ifnet.py @@ -0,0 +1,134 @@ +import os +import sys +import torch +import torch.nn as nn +import torch.nn.functional as F + +sys.path.append(os.path.dirname(__file__)) +from warplayer import warp # pylint: disable=wrong-import-position + + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1): + return nn.Sequential( + nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, + padding=padding, dilation=dilation, bias=True), + nn.LeakyReLU(0.2, True) + ) + +def conv_bn(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1): + return nn.Sequential( + nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, + padding=padding, dilation=dilation, bias=False), + nn.BatchNorm2d(out_planes), + nn.LeakyReLU(0.2, True) + ) + +class ResConv(nn.Module): + def __init__(self, c, dilation=1): + super(ResConv, self).__init__() + self.conv = nn.Conv2d(c, c, 3, 1, dilation, dilation=dilation, groups=1\ +) + self.beta = nn.Parameter(torch.ones((1, c, 1, 1)), requires_grad=True) + self.relu = nn.LeakyReLU(0.2, True) + + def forward(self, x): + return self.relu(self.conv(x) * self.beta + x) + +class IFBlock(nn.Module): + def __init__(self, in_planes, c=64): + super(IFBlock, self).__init__() + self.conv0 = nn.Sequential( + conv(in_planes, c//2, 3, 2, 1), + conv(c//2, c, 3, 2, 1), + ) + self.convblock = nn.Sequential( + ResConv(c), + ResConv(c), + ResConv(c), + ResConv(c), + ResConv(c), + ResConv(c), + ResConv(c), + ResConv(c), + ) + self.lastconv = nn.Sequential( + nn.ConvTranspose2d(c, 4*6, 4, 2, 1), + nn.PixelShuffle(2) + ) + + def forward(self, x, flow=None, scale=1): + x = F.interpolate(x, scale_factor= 1. / scale, mode="bilinear", align_corners=False) + if flow is not None: + flow = F.interpolate(flow, scale_factor= 1. / scale, mode="bilinear", align_corners=False) * 1. / scale + x = torch.cat((x, flow), 1) + feat = self.conv0(x) + feat = self.convblock(feat) + tmp = self.lastconv(feat) + tmp = F.interpolate(tmp, scale_factor=scale, mode="bilinear", align_corners=False) + flow = tmp[:, :4] * scale + mask = tmp[:, 4:5] + return flow, mask + +class IFNet(nn.Module): + def __init__(self): + super(IFNet, self).__init__() + self.block0 = IFBlock(7, c=192) + self.block1 = IFBlock(8+4, c=128) + self.block2 = IFBlock(8+4, c=96) + self.block3 = IFBlock(8+4, c=64) + # self.contextnet = Contextnet() + # self.unet = Unet() + + def forward( self, x, timestep=0.5, scale_list=[8, 4, 2, 1], training=False, fastmode=True, ensemble=False): # pylint: disable=dangerous-default-value # noqa: B006 + if training is False: + channel = x.shape[1] // 2 + img0 = x[:, :channel] + img1 = x[:, channel:] + if not torch.is_tensor(timestep): + timestep = (x[:, :1].clone() * 0 + 1) * timestep + else: + timestep = timestep.repeat(1, 1, img0.shape[2], img0.shape[3]) + flow_list = [] + merged = [] + mask_list = [] + warped_img0 = img0 + warped_img1 = img1 + flow = None + mask = None + # loss_cons = 0 + block = [self.block0, self.block1, self.block2, self.block3] + for i in range(4): + if flow is None: + flow, mask = block[i](torch.cat((img0[:, :3], img1[:, :3], timestep), 1), None, scale=scale_list[i]) + if ensemble: + f1, m1 = block[i](torch.cat((img1[:, :3], img0[:, :3], 1-timestep), 1), None, scale=scale_list[i]) + flow = (flow + torch.cat((f1[:, 2:4], f1[:, :2]), 1)) / 2 + mask = (mask + (-m1)) / 2 + else: + f0, m0 = block[i](torch.cat((warped_img0[:, :3], warped_img1[:, :3], timestep, mask), 1), flow, scale=scale_list[i]) + if ensemble: + f1, m1 = block[i](torch.cat((warped_img1[:, :3], warped_img0[:, :3], 1-timestep, -mask), 1), torch.cat((flow[:, 2:4], flow[:, :2]), 1), scale=scale_list[i]) # pylint: disable=invalid-unary-operand-type + f0 = (f0 + torch.cat((f1[:, 2:4], f1[:, :2]), 1)) / 2 + m0 = (m0 + (-m1)) / 2 + flow = flow + f0 + mask = mask + m0 + mask_list.append(mask) + flow_list.append(flow) + warped_img0 = warp(img0, flow[:, :2]) + warped_img1 = warp(img1, flow[:, 2:4]) + merged.append((warped_img0, warped_img1)) + mask_list[3] = torch.sigmoid(mask_list[3]) + merged[3] = merged[3][0] * mask_list[3] + merged[3][1] * (1 - mask_list[3]) + if not fastmode: + print('contextnet is removed') + ''' + c0 = self.contextnet(img0, flow[:, :2]) + c1 = self.contextnet(img1, flow[:, 2:4]) + tmp = self.unet(img0, img1, warped_img0, warped_img1, mask, flow, c0, c1) + res = tmp[:, :3] * 2 - 1 + merged[3] = torch.clamp(merged[3] + res, 0, 1) + ''' + return flow_list, mask_list[3], merged diff --git a/modules/rife/model_rife.py b/modules/rife/model_rife.py new file mode 100644 index 000000000..ce8c363cd --- /dev/null +++ b/modules/rife/model_rife.py @@ -0,0 +1,79 @@ +import torch +from torch.optim import AdamW +from torch.nn.parallel import DistributedDataParallel as DDP +from modules.rife.model_ifnet import IFNet +from modules.rife.loss import EPE, SOBEL +from modules import devices + + +class Model: + def __init__(self, local_rank=-1): + self.flownet = IFNet() + self.device() + self.optimG = AdamW(self.flownet.parameters(), lr=1e-6, weight_decay=1e-4) + self.epe = EPE() + self.version = 3.9 + # self.vgg = VGGPerceptualLoss().to(device) + self.sobel = SOBEL() + if local_rank != -1: + self.flownet = DDP(self.flownet, device_ids=[local_rank], output_device=local_rank) + + def train(self): + self.flownet.train() + + def eval(self): + self.flownet.eval() + + def device(self): + self.flownet.to(devices.device) + self.flownet.to(devices.dtype) + + def load_model(self, model_file, rank=0): + def convert(param): + if rank == -1: + return { k.replace("module.", ""): v for k, v in param.items() if "module." in k } + else: + return param + if rank <= 0: + if torch.cuda.is_available(): + self.flownet.load_state_dict(convert(torch.load(model_file)), False) + else: + self.flownet.load_state_dict(convert(torch.load(model_file, map_location='cpu')), False) + + def save_model(self, model_file, rank=0): + if rank == 0: + torch.save(self.flownet.state_dict(), model_file) + + def inference(self, img0, img1, timestep=0.5, scale=1.0): + imgs = torch.cat((img0, img1), 1) + scale_list = [8/scale, 4/scale, 2/scale, 1/scale] + _flow, _mask, merged = self.flownet(imgs, timestep, scale_list) + return merged[3] + + def update(self, imgs, gt, learning_rate=0, mul=1, training=True, flow_gt=None): # pylint: disable=unused-argument + for param_group in self.optimG.param_groups: + param_group['lr'] = learning_rate + # img0 = imgs[:, :3] + # img1 = imgs[:, 3:] + if training: + self.train() + else: + self.eval() + scale = [8, 4, 2, 1] + flow, mask, merged = self.flownet(torch.cat((imgs, gt), 1), scale=scale, training=training) + loss_l1 = (merged[3] - gt).abs().mean() + loss_smooth = self.sobel(flow[3], flow[3]*0).mean() + # loss_vgg = self.vgg(merged[2], gt) + if training: + self.optimG.zero_grad() + loss_G = loss_l1 + loss_smooth * 0.1 + loss_G.backward() + self.optimG.step() + # else: + # flow_teacher = flow[2] + return merged[3], { + 'mask': mask, + 'flow': flow[3][:, :2], + 'loss_l1': loss_l1, + 'loss_smooth': loss_smooth, + } diff --git a/modules/rife/refine.py b/modules/rife/refine.py new file mode 100644 index 000000000..a66488345 --- /dev/null +++ b/modules/rife/refine.py @@ -0,0 +1,90 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from modules.rife.warplayer import warp + + +c = 16 + + +def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1): + return nn.Sequential( + nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=True), + nn.LeakyReLU(0.2, True) + ) + + +def conv_woact(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1): + return nn.Sequential( + nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=True), + ) + + +def deconv(in_planes, out_planes, kernel_size=4, stride=2, padding=1): # pylint: disable=unused-argument + return nn.Sequential( + torch.nn.ConvTranspose2d(in_channels=in_planes, out_channels=out_planes, kernel_size=4, stride=2, padding=1, bias=True), + nn.LeakyReLU(0.2, True) + ) + + +class Conv2(nn.Module): + def __init__(self, in_planes, out_planes, stride=2): + super(Conv2, self).__init__() + self.conv1 = conv(in_planes, out_planes, 3, stride, 1) + self.conv2 = conv(out_planes, out_planes, 3, 1, 1) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + return x + + +class Contextnet(nn.Module): + def __init__(self): + super(Contextnet, self).__init__() + self.conv1 = Conv2(3, c) + self.conv2 = Conv2(c, 2*c) + self.conv3 = Conv2(2*c, 4*c) + self.conv4 = Conv2(4*c, 8*c) + + def forward(self, x, flow): + x = self.conv1(x) + flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False) * 0.5 + f1 = warp(x, flow) + x = self.conv2(x) + flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False) * 0.5 + f2 = warp(x, flow) + x = self.conv3(x) + flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False) * 0.5 + f3 = warp(x, flow) + x = self.conv4(x) + flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False) * 0.5 + f4 = warp(x, flow) + return [f1, f2, f3, f4] + + +class Unet(nn.Module): + def __init__(self): + super(Unet, self).__init__() + self.down0 = Conv2(17, 2*c) + self.down1 = Conv2(4*c, 4*c) + self.down2 = Conv2(8*c, 8*c) + self.down3 = Conv2(16*c, 16*c) + self.up0 = deconv(32*c, 8*c) + self.up1 = deconv(16*c, 4*c) + self.up2 = deconv(8*c, 2*c) + self.up3 = deconv(4*c, c) + self.conv = nn.Conv2d(c, 3, 3, 1, 1) + + def forward(self, img0, img1, warped_img0, warped_img1, mask, flow, c0, c1): + s0 = self.down0( + torch.cat((img0, img1, warped_img0, warped_img1, mask, flow), 1)) + s1 = self.down1(torch.cat((s0, c0[0], c1[0]), 1)) + s2 = self.down2(torch.cat((s1, c0[1], c1[1]), 1)) + s3 = self.down3(torch.cat((s2, c0[2], c1[2]), 1)) + x = self.up0(torch.cat((s3, c0[3], c1[3]), 1)) + x = self.up1(torch.cat((x, s2), 1)) + x = self.up2(torch.cat((x, s1), 1)) + x = self.up3(torch.cat((x, s0), 1)) + x = self.conv(x) + return torch.sigmoid(x) diff --git a/modules/rife/ssim.py b/modules/rife/ssim.py new file mode 100644 index 000000000..e2261ca7a --- /dev/null +++ b/modules/rife/ssim.py @@ -0,0 +1,174 @@ +from math import exp +import torch +import torch.nn.functional as F +from modules import devices + + +def gaussian(window_size, sigma): + gauss = torch.Tensor([exp(-(x - window_size//2)**2/float(2*sigma**2)) for x in range(window_size)]) + return gauss/gauss.sum() + + +def create_window(window_size, channel=1): + _1D_window = gaussian(window_size, 1.5).unsqueeze(1) + _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0).to(devices.device) + window = _2D_window.expand(channel, 1, window_size, window_size).contiguous() + return window + + +def create_window_3d(window_size, channel=1): + _1D_window = gaussian(window_size, 1.5).unsqueeze(1) + _2D_window = _1D_window.mm(_1D_window.t()) + _3D_window = _2D_window.unsqueeze(2) @ (_1D_window.t()) + window = _3D_window.expand(1, channel, window_size, window_size, window_size).contiguous().to(devices.device) + return window + + +def ssim(img1, img2, window_size=11, window=None, size_average=True, full=False, val_range=None): + # Value range can be different from 255. Other common ranges are 1 (sigmoid) and 2 (tanh). + if val_range is None: + if torch.max(img1) > 128: + max_val = 255 + else: + max_val = 1 + + if torch.min(img1) < -0.5: + min_val = -1 + else: + min_val = 0 + L = max_val - min_val + else: + L = val_range + padd = 0 + (_, channel, height, width) = img1.size() + if window is None: + real_size = min(window_size, height, width) + window = create_window(real_size, channel=channel).to(img1.device) + # mu1 = F.conv2d(img1, window, padding=padd, groups=channel) + # mu2 = F.conv2d(img2, window, padding=padd, groups=channel) + mu1 = F.conv2d(F.pad(img1, (5, 5, 5, 5), mode='replicate'), window, padding=padd, groups=channel) + mu2 = F.conv2d(F.pad(img2, (5, 5, 5, 5), mode='replicate'), window, padding=padd, groups=channel) + mu1_sq = mu1.pow(2) + mu2_sq = mu2.pow(2) + mu1_mu2 = mu1 * mu2 + sigma1_sq = F.conv2d(F.pad(img1 * img1, (5, 5, 5, 5), 'replicate'), window, padding=padd, groups=channel) - mu1_sq + sigma2_sq = F.conv2d(F.pad(img2 * img2, (5, 5, 5, 5), 'replicate'), window, padding=padd, groups=channel) - mu2_sq + sigma12 = F.conv2d(F.pad(img1 * img2, (5, 5, 5, 5), 'replicate'), window, padding=padd, groups=channel) - mu1_mu2 + C1 = (0.01 * L) ** 2 + C2 = (0.03 * L) ** 2 + v1 = 2.0 * sigma12 + C2 + v2 = sigma1_sq + sigma2_sq + C2 + cs = torch.mean(v1 / v2) # contrast sensitivity + ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2) + if size_average: + ret = ssim_map.mean() + else: + ret = ssim_map.mean(1).mean(1).mean(1) + if full: + return ret, cs + return ret + + +def ssim_matlab(img1, img2, window_size=11, window=None, size_average=True, full=False, val_range=None): + # Value range can be different from 255. Other common ranges are 1 (sigmoid) and 2 (tanh). + if val_range is None: + if torch.max(img1) > 128: + max_val = 255 + else: + max_val = 1 + if torch.min(img1) < -0.5: + min_val = -1 + else: + min_val = 0 + L = max_val - min_val + else: + L = val_range + padd = 0 + (_, _, height, width) = img1.size() + if window is None: + real_size = min(window_size, height, width) + window = create_window_3d(real_size, channel=1).to(img1.device) + # Channel is set to 1 since we consider color images as volumetric images + img1 = img1.unsqueeze(1) + img2 = img2.unsqueeze(1) + mu1 = F.conv3d(F.pad(img1, (5, 5, 5, 5, 5, 5), mode='replicate'), window, padding=padd, groups=1) + mu2 = F.conv3d(F.pad(img2, (5, 5, 5, 5, 5, 5), mode='replicate'), window, padding=padd, groups=1) + mu1_sq = mu1.pow(2) + mu2_sq = mu2.pow(2) + mu1_mu2 = mu1 * mu2 + sigma1_sq = F.conv3d(F.pad(img1 * img1, (5, 5, 5, 5, 5, 5), 'replicate'), window, padding=padd, groups=1) - mu1_sq + sigma2_sq = F.conv3d(F.pad(img2 * img2, (5, 5, 5, 5, 5, 5), 'replicate'), window, padding=padd, groups=1) - mu2_sq + sigma12 = F.conv3d(F.pad(img1 * img2, (5, 5, 5, 5, 5, 5), 'replicate'), window, padding=padd, groups=1) - mu1_mu2 + C1 = (0.01 * L) ** 2 + C2 = (0.03 * L) ** 2 + v1 = 2.0 * sigma12 + C2 + v2 = sigma1_sq + sigma2_sq + C2 + cs = torch.mean(v1 / v2) # contrast sensitivity + ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2) + if size_average: + ret = ssim_map.mean() + else: + ret = ssim_map.mean(1).mean(1).mean(1) + if full: + return ret, cs + return ret + + +def msssim(img1, img2, window_size=11, size_average=True, val_range=None, normalize=False): + local_device = img1.device + weights = torch.FloatTensor([0.0448, 0.2856, 0.3001, 0.2363, 0.1333]).to(local_device) + levels = weights.size()[0] + mssim = [] + mcs = [] + for _ in range(levels): + sim, cs = ssim(img1, img2, window_size=window_size, size_average=size_average, full=True, val_range=val_range) + mssim.append(sim) + mcs.append(cs) + img1 = F.avg_pool2d(img1, (2, 2)) + img2 = F.avg_pool2d(img2, (2, 2)) + mssim = torch.stack(mssim) + mcs = torch.stack(mcs) + # Normalize (to avoid NaNs during training unstable models, not compliant with original definition) + if normalize: + mssim = (mssim + 1) / 2 + mcs = (mcs + 1) / 2 + pow1 = mcs ** weights + pow2 = mssim ** weights + # From Matlab implementation https://ece.uwaterloo.ca/~z70wang/research/iwssim/ + output = torch.prod(pow1[:-1] * pow2[-1]) + return output + + +# Classes to re-use window +class SSIM(torch.nn.Module): + def __init__(self, window_size=11, size_average=True, val_range=None): + super(SSIM, self).__init__() + self.window_size = window_size + self.size_average = size_average + self.val_range = val_range + # Assume 3 channel for SSIM + self.channel = 3 + self.window = create_window(window_size, channel=self.channel) + + def forward(self, img1, img2): + (_, channel, _, _) = img1.size() + if channel == self.channel and self.window.dtype == img1.dtype: + window = self.window + else: + window = create_window(self.window_size, channel).to(img1.device).type(img1.dtype) + self.window = window + self.channel = channel + _ssim = ssim(img1, img2, window=window, window_size=self.window_size, size_average=self.size_average) + dssim = (1 - _ssim) / 2 + return dssim + + +class MSSSIM(torch.nn.Module): + def __init__(self, window_size=11, size_average=True, channel=3): + super(MSSSIM, self).__init__() + self.window_size = window_size + self.size_average = size_average + self.channel = channel + + def forward(self, img1, img2): + return msssim(img1, img2, window_size=self.window_size, size_average=self.size_average) diff --git a/modules/rife/warplayer.py b/modules/rife/warplayer.py new file mode 100644 index 000000000..abcd6aa5c --- /dev/null +++ b/modules/rife/warplayer.py @@ -0,0 +1,17 @@ +import torch +from modules import devices + + +backwarp_tenGrid = {} + + +def warp(tenInput, tenFlow): + k = (str(tenFlow.device), str(tenFlow.size())) + if k not in backwarp_tenGrid: + tenHorizontal = torch.linspace(-1.0, 1.0, tenFlow.shape[3], device=devices.device).view(1, 1, 1, tenFlow.shape[3]).expand(tenFlow.shape[0], -1, tenFlow.shape[2], -1) + tenVertical = torch.linspace(-1.0, 1.0, tenFlow.shape[2], device=devices.device).view(1, 1, tenFlow.shape[2], 1).expand(tenFlow.shape[0], -1, -1, tenFlow.shape[3]) + backwarp_tenGrid[k] = torch.cat([tenHorizontal, tenVertical], 1).to(devices.device) + tenFlow = torch.cat([tenFlow[:, 0:1, :, :] / ((tenInput.shape[3] - 1.0) / 2.0), + tenFlow[:, 1:2, :, :] / ((tenInput.shape[2] - 1.0) / 2.0)], 1) + grid = (backwarp_tenGrid[k] + tenFlow).permute(0, 2, 3, 1).to(devices.dtype) + return torch.nn.functional.grid_sample(input=tenInput, grid=grid, mode='bilinear', padding_mode='border', align_corners=True) diff --git a/modules/script_callbacks.py b/modules/script_callbacks.py index e30f88dd1..663199968 100644 --- a/modules/script_callbacks.py +++ b/modules/script_callbacks.py @@ -329,7 +329,7 @@ def before_ui_callback(): def add_callback(callbacks, fun): - stack = [x for x in inspect.stack() if x.filename != __file__] + stack = [x for x in inspect.stack(0) if x.filename != __file__] filename = stack[0].filename if len(stack) > 0 else 'unknown file' callbacks.append(ScriptCallback(filename, fun)) diff --git a/modules/script_loading.py b/modules/script_loading.py index 5ac82f538..b20d4072e 100644 --- a/modules/script_loading.py +++ b/modules/script_loading.py @@ -3,7 +3,7 @@ import os import contextlib import importlib.util import modules.errors as errors -from installer import setup_logging +from installer import setup_logging, args preloaded = [] @@ -12,6 +12,10 @@ preloaded = [] def load_module(path): module_spec = importlib.util.spec_from_file_location(os.path.basename(path), path) module = importlib.util.module_from_spec(module_spec) + if args.profile: + import cProfile + pr = cProfile.Profile() + pr.enable() try: if '/sd-extension-' in path: # safe extensions without stdout intercept module_spec.loader.exec_module(module) @@ -25,6 +29,8 @@ def load_module(path): errors.log.info(f"Extension: script='{os.path.relpath(path)}' {line.strip()}") except Exception as e: errors.display(e, f'Module load: {path}') + if args.profile: + errors.profile(pr, f'Scripts: {path}') return module diff --git a/modules/scripts.py b/modules/scripts.py index 01c501f2c..aee3545b2 100644 --- a/modules/scripts.py +++ b/modules/scripts.py @@ -261,6 +261,7 @@ def load_scripts(): elif issubclass(script_class, scripts_postprocessing.ScriptPostprocessing): postprocessing_scripts_data.append(ScriptClassData(script_class, scriptfile.path, scriptfile.basedir, module)) + from installer import args for scriptfile in scripts_list: try: if scriptfile.basedir != paths.script_path: @@ -274,12 +275,10 @@ def load_scripts(): current_basedir = paths.script_path t.record(os.path.basename(scriptfile.basedir)) sys.path = syspath - global scripts_txt2img, scripts_img2img, scripts_postproc # pylint: disable=global-statement scripts_txt2img = ScriptRunner() scripts_img2img = ScriptRunner() scripts_postproc = scripts_postprocessing.ScriptPostprocessingRunner() - return t, time.time()-t0 diff --git a/modules/sd_models.py b/modules/sd_models.py index add83e11a..0956353ce 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -20,7 +20,7 @@ import tomesd from transformers import logging as transformers_logging import ldm.modules.midas as midas from ldm.util import instantiate_from_config -from modules import paths, shared, shared_items, shared_state, modelloader, devices, script_callbacks, sd_vae, sd_disable_initialization, errors, hashes, sd_models_config, sd_models_compile, sd_hijack_inpainting +from modules import paths, shared, shared_items, shared_state, modelloader, devices, script_callbacks, sd_vae, errors, hashes, sd_models_config, sd_models_compile, sd_hijack_inpainting from modules.timer import Timer from modules.memstats import memory_stats from modules.paths import models_path, script_path @@ -603,6 +603,7 @@ def detect_pipeline(f: str, op: str = 'model', warning=True): warn = shared.log.warning if warning else lambda *args, **kwargs: None if guess == 'Autodetect': try: + # guess by size size = round(os.path.getsize(f) / 1024 / 1024) if size < 128: warn(f'Model size smaller than expected: {f} size={size} MB') @@ -635,14 +636,27 @@ def detect_pipeline(f: str, op: str = 'model', warning=True): guess = 'Stable Diffusion XL Instruct' else: guess = 'Stable Diffusion' - if 'LCM_' in f or 'LCM-' in f: + # guess by name + """ + if 'LCM_' in f.upper() or 'LCM-' in f.upper() or '_LCM' in f.upper() or '-LCM' in f.upper(): if shared.backend == shared.Backend.ORIGINAL: warn(f'Model detected as LCM model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'Latent Consistency Model' + """ if 'PixArt' in f: if shared.backend == shared.Backend.ORIGINAL: warn(f'Model detected as PixArt Alpha model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'PixArt Alpha' + # switch for specific variant + if guess == 'Stable Diffusion' and 'inpaint' in f.lower(): + guess = 'Stable Diffusion Inpaint' + elif guess == 'Stable Diffusion' and 'instruct' in f.lower(): + guess = 'Stable Diffusion Instruct' + if guess == 'Stable Diffusion XL' and 'inpaint' in f.lower(): + guess = 'Stable Diffusion XL Inpaint' + elif guess == 'Stable Diffusion XL' and 'instruct' in f.lower(): + guess = 'Stable Diffusion XL Instruct' + # get actual pipeline pipeline = shared_items.get_pipelines().get(guess, None) shared.log.info(f'Autodetect: {op}="{guess}" class={pipeline.__name__} file="{f}" size={size}MB') except Exception as e: @@ -729,13 +743,13 @@ def set_diffuser_options(sd_model, vae, op: str): sd_model.enable_xformers_memory_efficient_attention() if shared.opts.diffusers_eval: - if hasattr(sd_model, "unet"): + if hasattr(sd_model, "unet") and hasattr(sd_model.unet, "requires_grad_"): sd_model.unet.requires_grad_(False) sd_model.unet.eval() - if hasattr(sd_model, "vae"): + if hasattr(sd_model, "vae") and hasattr(sd_model.vae, "requires_grad_"): sd_model.vae.requires_grad_(False) sd_model.vae.eval() - if hasattr(sd_model, "text_encoder"): + if hasattr(sd_model, "text_encoder") and hasattr(sd_model.text_encoder, "requires_grad_"): sd_model.text_encoder.requires_grad_(False) sd_model.text_encoder.eval() @@ -746,6 +760,10 @@ def set_diffuser_options(sd_model, vae, op: str): def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument import torch # pylint: disable=reimported,redefined-outer-name + if shared.cmd_opts.profile: + import cProfile + pr = cProfile.Profile() + pr.enable() if timer is None: timer = Timer() logging.getLogger("diffusers").setLevel(logging.ERROR) @@ -888,6 +906,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No shared.log.debug(f'Setting {op}: pipeline={sd_model.__class__.__name__} config={diffusers_load_config}') # pylint: disable=protected-access except Exception as e: shared.log.error(f'Diffusers failed loading: {op}={checkpoint_info.path} pipeline={shared.opts.diffusers_pipeline}/{sd_model.__class__.__name__} {e}') + errors.display(e, f'loading {op}={checkpoint_info.path} pipeline={shared.opts.diffusers_pipeline}/{sd_model.__class__.__name__}') return else: shared.log.error(f'Diffusers cannot load: {op}={checkpoint_info.path}') @@ -961,6 +980,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No timer.record("load") devices.torch_gc(force=True) + if shared.cmd_opts.profile: + errors.profile(pr, 'Load') script_callbacks.model_loaded_callback(sd_model) shared.log.info(f"Load {op}: time={timer.summary()} native={get_native(sd_model)} {memory_stats()}") @@ -1093,12 +1114,16 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, sd_model = None stdout = io.StringIO() with contextlib.redirect_stdout(stdout): + """ try: clip_is_included_into_sd = sd1_clip_weight in state_dict or sd2_clip_weight in state_dict with sd_disable_initialization.DisableInitialization(disable_clip=clip_is_included_into_sd): sd_model = instantiate_from_config(sd_config.model) - except Exception: + except Exception as e: + shared.log.error(f'LDM: instantiate from config: {e}') sd_model = instantiate_from_config(sd_config.model) + """ + sd_model = instantiate_from_config(sd_config.model) for line in stdout.getvalue().splitlines(): if len(line) > 0: shared.log.info(f'LDM: {line.strip()}') diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index dfac73eb2..dd24d42ae 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -31,12 +31,15 @@ def setup_img2img_steps(p, steps=None): def single_sample_to_image(sample, approximation=None): + # sample should be [4,64,64] if approximation is None: approximation = approximation_indexes.get(shared.opts.show_progress_type, None) if approximation is None: warn_once('Unknown decode type, please reset preview method') approximation = 0 + if len(sample.shape) == 4 and sample.shape[0]: # likely animatediff latent + sample = sample.permute(1, 0, 2, 3)[0] if approximation == 0: # Simple x_sample = sd_vae_approx.cheap_approximation(sample) * 0.5 + 0.5 elif approximation == 1: # Approximate diff --git a/modules/shared.py b/modules/shared.py index 176bd4690..7c2bea684 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -159,7 +159,7 @@ def list_samplers(): def temp_disable_extensions(): disable_safe = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-agent-scheduler', 'clip-interrogator-ext', 'stable-diffusion-webui-rembg', 'sd-extension-chainner', 'stable-diffusion-webui-images-browser'] - disable_diffusers = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris'] + disable_diffusers = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-animatediff'] disable_original = [] disabled = [] if cmd_opts.safe: @@ -411,7 +411,7 @@ options_templates.update(options_section(('saving-images', "Image Options"), { "image_sep_metadata": OptionInfo("

Metadata/Logging

", "", gr.HTML), "image_metadata": OptionInfo(True, "Include metadata in saved images"), - "save_txt": OptionInfo(False, "Create text file next to every image with generation parameters"), + "save_txt": OptionInfo(False, "Create info file for each every image"), "save_log_fn": OptionInfo("", "Create JSON log file for each saved image", component_args=hide_dirs), "image_watermark_enabled": OptionInfo(False, "Include watermark in saved images"), "image_watermark": OptionInfo('', "Image watermark string"), @@ -446,11 +446,12 @@ options_templates.update(options_section(('saving-paths', "Image Naming & Paths" "outdir_sep_dirs": OptionInfo("

Directories

", "", gr.HTML), "outdir_samples": OptionInfo("", "Output directory for images", component_args=hide_dirs, folder=True), - "outdir_txt2img_samples": OptionInfo("outputs/text", 'Output directory for txt2img images', component_args=hide_dirs, folder=True), - "outdir_img2img_samples": OptionInfo("outputs/image", 'Output directory for img2img images', component_args=hide_dirs, folder=True), - "outdir_extras_samples": OptionInfo("outputs/extras", 'Output directory for images from extras tab', component_args=hide_dirs, folder=True), - "outdir_save": OptionInfo("outputs/save", "Directory for saving images using the Save button", component_args=hide_dirs, folder=True), - "outdir_init_images": OptionInfo("outputs/init-images", "Directory for saving init images when using img2img", component_args=hide_dirs, folder=True), + "outdir_txt2img_samples": OptionInfo("outputs/text", 'Directory for text generate', component_args=hide_dirs, folder=True), + "outdir_img2img_samples": OptionInfo("outputs/image", 'Directory for image generate', component_args=hide_dirs, folder=True), + "outdir_extras_samples": OptionInfo("outputs/extras", 'Directory for processed images', component_args=hide_dirs, folder=True), + "outdir_save": OptionInfo("outputs/save", "Directory for manually saved images", component_args=hide_dirs, folder=True), + "outdir_video": OptionInfo("outputs/video", "Directory for videos", component_args=hide_dirs, folder=True), + "outdir_init_images": OptionInfo("outputs/init-images", "Directory for init images", component_args=hide_dirs, folder=True), "outdir_sep_grids": OptionInfo("

Grids

", "", gr.HTML), "grid_extended_filename": OptionInfo(True, "Add extended info (seed, prompt) to filename when saving grid", gr.Checkbox, {"visible": False}), @@ -609,7 +610,7 @@ options_templates.update(options_section(('extra_networks', "Extra Networks"), { "extra_networks_styles": OptionInfo(True, "Show built-in styles"), "lora_preferred_name": OptionInfo("filename", "LoRA preffered name", gr.Radio, {"choices": ["filename", "alias"]}), "lora_add_hashes_to_infotext": OptionInfo(True, "LoRA add hash info"), - "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}), + "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 24, "step": 1}), "lora_functional": OptionInfo(False, "Use Kohya method for handling multiple LoRA", gr.Checkbox, { "visible": False }), "sd_hypernetwork": OptionInfo("None", "Add hypernetwork to prompt", gr.Dropdown, { "choices": ["None"], "visible": False }), @@ -780,6 +781,7 @@ class Options: value = expected_type(value) return value +profiler = None opts = Options() config_filename = cmd_opts.config opts.load(config_filename) @@ -803,6 +805,7 @@ device = devices.device batch_cond_uncond = opts.always_batch_cond_uncond or not (cmd_opts.lowvram or cmd_opts.medvram) parallel_processing_allowed = not cmd_opts.lowvram mem_mon = modules.memmon.MemUsageMonitor("MemMon", devices.device) +max_workers = 2 if devices.backend == "directml": directml_do_hijack() @@ -958,6 +961,8 @@ class Shared(sys.modules[__name__].__class__): # this class is here to provide s model_type = 'sd' elif "LatentConsistencyModel" in self.sd_model.__class__.__name__: model_type = 'sd' # lcm is compatible with sd + elif "AnimateDiffPipeline" in self.sd_model.__class__.__name__: + model_type = 'sd' # ad is compatible with sd elif "Kandinsky" in self.sd_model.__class__.__name__: model_type = 'kandinsky' else: diff --git a/modules/shared_items.py b/modules/shared_items.py index 72d8cbfca..b0d4f1ec4 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -30,6 +30,7 @@ def get_pipelines(): pipelines = { # note: not all pipelines can be used manually as they require prior pipeline next to decoder pipeline 'Autodetect': None, 'Stable Diffusion': getattr(diffusers, 'StableDiffusionPipeline', None), + 'Stable Diffusion Inpaint': getattr(diffusers, 'StableDiffusionInpaintPipeline', None), 'Stable Diffusion Img2Img': getattr(diffusers, 'StableDiffusionImg2ImgPipeline', None), 'Stable Diffusion Instruct': getattr(diffusers, 'StableDiffusionInstructPix2PixPipeline', None), 'Stable Diffusion Upscale': getattr(diffusers, 'StableDiffusionUpscalePipeline', None), @@ -43,7 +44,7 @@ def get_pipelines(): 'Wuerstchen': getattr(diffusers, 'WuerstchenCombinedPipeline', None), 'Kandinsky 2.1': getattr(diffusers, 'KandinskyPipeline', None), 'Kandinsky 2.2': getattr(diffusers, 'KandinskyV22Pipeline', None), - # 'Kandinsky 3': getattr(diffusers, 'KandinskyV3Pipeline', None), + 'Kandinsky 3': getattr(diffusers, 'Kandinsky3Pipeline', None), 'DeepFloyd IF': getattr(diffusers, 'IFPipeline', None), 'Custom Diffusers Pipeline': getattr(diffusers, 'DiffusionPipeline', None), # Segmind SSD-1B, Segmind Tiny diff --git a/modules/styles.py b/modules/styles.py index 427e76ea9..1f9a6fd52 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -4,6 +4,7 @@ import re import os import csv import json +import time from installer import log @@ -60,7 +61,7 @@ def apply_styles_to_extra(p, style: Style): v = type(orig)(v) setattr(p, k, v) fields.append(f'{k}={v}') - log.info(f'Applying style: name={style.name} extra={fields}') + log.info(f'Applying style: name="{style.name}" extra={fields}') class StyleDatabase: @@ -87,6 +88,7 @@ class StyleDatabase: def load_style(self, fn, prefix=None): with open(fn, 'r', encoding='utf-8') as f: + new_style = None try: all_styles = json.load(f) if type(all_styles) is dict: @@ -100,7 +102,7 @@ class StyleDatabase: name = os.path.join(prefix, name) else: name = os.path.join(os.path.dirname(os.path.relpath(fn, self.path)), name) - self.styles[style["name"]] = Style( + new_style = Style( name=name, desc=style.get('description', name), prompt=style.get("prompt", ""), @@ -110,26 +112,37 @@ class StyleDatabase: filename=fn, mtime=os.path.getmtime(fn), ) + self.styles[style["name"]] = new_style except Exception as e: log.error(f'Failed to load style: file={fn} error={e}') + return new_style def reload(self): + t0 = time.time() self.styles.clear() + def list_folder(folder): - for filename in os.listdir(folder): - fn = os.path.abspath(os.path.join(folder, filename)) - if os.path.isfile(fn) and fn.lower().endswith(".json"): - self.load_style(fn) - elif os.path.isdir(fn) and not fn.startswith('.'): - list_folder(fn) + import concurrent + future_items = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + for filename in os.listdir(folder): + fn = os.path.abspath(os.path.join(folder, filename)) + if os.path.isfile(fn) and fn.lower().endswith(".json"): + future_items[executor.submit(self.load_style, fn, None)] = fn + # self.load_style(fn) + elif os.path.isdir(fn) and not fn.startswith('.'): + list_folder(fn) + self.styles = dict(sorted(self.styles.items(), key=lambda style: style[1].filename)) + if self.built_in: + fn = os.path.join('html', 'art-styles.json') + future_items[executor.submit(self.load_style, fn, 'built-in')] = fn + for future in concurrent.futures.as_completed(future_items): + future.result() list_folder(self.path) - self.styles = dict(sorted(self.styles.items(), key=lambda style: style[1].filename)) - if self.built_in: - self.load_style(os.path.join('html', 'art-styles.json'), 'built-in') - - log.debug(f'Load styles: folder="{self.path}" items={len(self.styles.keys())}') + t1 = time.time() + log.debug(f'Load styles: folder="{self.path}" items={len(self.styles.keys())} time={t1-t0:.2f}') def find_style(self, name): found = [style for style in self.styles.values() if style.name == name] diff --git a/modules/txt2img.py b/modules/txt2img.py index 899af07db..55751bb2d 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -4,7 +4,7 @@ from modules.generation_parameters_copypaste import create_override_settings_dic from modules.ui import plaintext_to_html -def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, full_quality: bool, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_force: bool, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_steps: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, override_settings_texts, *args): # pylint: disable=unused-argument +def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, full_quality: bool, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_force: bool, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_steps: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_center, hdr_max_boundry, override_settings_texts, *args): # pylint: disable=unused-argument shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|latent_index={latent_index}|full_quality={full_quality}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}||height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_force={hr_force}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|image_cfg_scale={image_cfg_scale}|diffusers_guidance_rescale={diffusers_guidance_rescale}|refiner_steps={refiner_steps}|refiner_start={refiner_start}|refiner_prompt={refiner_prompt}|refiner_negative={refiner_negative}|override_settings_texts={override_settings_texts}') @@ -57,6 +57,9 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step refiner_start=refiner_start, refiner_prompt=refiner_prompt, refiner_negative=refiner_negative, + hdr_clamp=hdr_clamp, hdr_boundary=hdr_boundary, hdr_threshold=hdr_threshold, + hdr_center=hdr_center, hdr_channel_shift=hdr_channel_shift, hdr_full_shift=hdr_full_shift, + hdr_maximize=hdr_maximize, hdr_max_center=hdr_max_center, hdr_max_boundry=hdr_max_boundry, override_settings=override_settings, ) p.scripts = modules.scripts.scripts_txt2img diff --git a/modules/ui.py b/modules/ui.py index 0dcf9c25f..6b076469b 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -332,7 +332,7 @@ def create_sampler_and_steps_selection(choices, tabname): with FormRow(elem_classes=['flex-break']): sampler_index = gr.Dropdown(label='Sampling method', elem_id=f"{tabname}_sampling", choices=[x.name for x in choices], value='Default', type="index") - steps = gr.Slider(minimum=0, maximum=99, step=1, label="Sampling steps", elem_id=f"{tabname}_steps", value=20) + steps = gr.Slider(minimum=1, maximum=99, step=1, label="Sampling steps", elem_id=f"{tabname}_steps", value=20) if modules.shared.backend == modules.shared.Backend.ORIGINAL: with FormRow(elem_classes=['flex-break']): choices = ['brownian noise', 'discard penultimate sigma'] @@ -354,7 +354,6 @@ def create_sampler_and_steps_selection(choices, tabname): values += ['low order'] if opts.data.get('schedulers_use_loworder', True) else [] sampler_options = gr.CheckboxGroup(label='Sampler options', choices=choices, value=values, type='value') sampler_options.change(fn=set_sampler_diffuser_options, inputs=[sampler_options], outputs=[]) - return steps, sampler_index @@ -417,16 +416,31 @@ def create_ui(startup_timer = None): seed, reuse_seed, subseed, reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w = create_seed_inputs('txt2img') with gr.Accordion(open=False, label="Advanced", elem_id="txt2img_advanced", elem_classes=["small-accordion"]): - with FormRow(): - cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id="txt2img_cfg_scale") - clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=14, step=1, elem_id='txt2img_clip_skip', interactive=True) - with FormRow(): - image_cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='Secondary CFG scale', value=6.0, elem_id="txt2img_image_cfg_scale") - diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id="txt2img_image_cfg_rescale") - with FormRow(): - full_quality = gr.Checkbox(label='Full quality', value=True, elem_id="txt2img_full_quality") - restore_faces = gr.Checkbox(label='Face restore', value=False, visible=len(modules.shared.face_restorers) > 1, elem_id="txt2img_restore_faces") - tiling = gr.Checkbox(label='Tiling', value=False, elem_id="txt2img_tiling") + with gr.Group(): + with FormRow(): + cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id="txt2img_cfg_scale") + clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=14, step=1, elem_id='txt2img_clip_skip', interactive=True) + with FormRow(): + image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Secondary CFG scale', value=6.0, elem_id="txt2img_image_cfg_scale") + diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id="txt2img_image_cfg_rescale") + with gr.Group(): + with FormRow(): + full_quality = gr.Checkbox(label='Full quality', value=True, elem_id="txt2img_full_quality") + restore_faces = gr.Checkbox(label='Face restore', value=False, visible=len(modules.shared.face_restorers) > 1, elem_id="txt2img_restore_faces") + tiling = gr.Checkbox(label='Tiling', value=False, elem_id="txt2img_tiling") + with gr.Group(): + with FormRow(): + hdr_clamp = gr.Checkbox(label='HDR clamp', value=False, elem_id="txt2img_hdr_clamp") + hdr_boundary = gr.Slider(minimum=0.0, maximum=10.0, step=0.1, value=4.0, label='Range', elem_id="txt2img_hdr_boundary") + hdr_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, value=0.95, label='Threshold', elem_id="txt2img_hdr_threshold") + with FormRow(): + hdr_center = gr.Checkbox(label='HDR center', value=False, elem_id="txt2img_hdr_center") + hdr_channel_shift = gr.Slider(minimum=0.0, maximum=2.0, step=0.1, value=1.0, label='Channel shift', elem_id="txt2img_hdr_channel_shift") + hdr_full_shift = gr.Slider(minimum=0.0, maximum=2.0, step=0.1, value=1, label='Full shift', elem_id="txt2img_hdr_full_shift") + with FormRow(): + hdr_maximize = gr.Checkbox(label='HDR maximize', value=False, elem_id="txt2img_hdr_maximize") + hdr_max_center = gr.Slider(minimum=0.0, maximum=2.0, step=0.1, value=0.6, label='Center', elem_id="txt2img_hdr_max_center") + hdr_max_boundry = gr.Slider(minimum=0.5, maximum=2.0, step=0.1, value=1.0, label='Range', elem_id="txt2img_hdr_max_boundry") with gr.Accordion(open=False, label="Second pass", elem_id="txt2img_second_pass", elem_classes=["small-accordion"]): with FormGroup(): @@ -493,6 +507,7 @@ def create_ui(startup_timer = None): enable_hr, denoising_strength, hr_scale, hr_upscaler, hr_force, hr_second_pass_steps, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative, + hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_center, hdr_max_boundry, override_settings, ] + custom_inputs, outputs=[ @@ -711,8 +726,8 @@ def create_ui(startup_timer = None): with gr.Accordion(open=False, label="Advanced", elem_classes=["small-accordion"], elem_id="img2img_advanced_group"): with FormRow(): - cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id="img2img_cfg_scale") - image_cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.15, label='Image CFG scale', value=1.5, elem_id="img2img_image_cfg_scale") + cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id="img2img_cfg_scale") + image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.15, label='Image CFG scale', value=1.5, elem_id="img2img_image_cfg_scale") with FormRow(): clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=4, step=1, elem_id='img2img_clip_skip', interactive=True) diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id="txt2img_image_cfg_rescale") diff --git a/modules/ui_common.py b/modules/ui_common.py index b1a5bf364..d63c1ce7a 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -35,16 +35,16 @@ def plaintext_to_html(text): def infotext_to_html(text): res = parse_generation_parameters(text) - prompt = res.get('Prompt', None) - negative = res.get('Negative prompt', None) + prompt = res.get('Prompt', '') + negative = res.get('Negative prompt', '') res.pop('Prompt', None) res.pop('Negative prompt', None) params = [f'{k}: {v}' for k, v in res.items() if v is not None] - params = '| '.join(params) + params = '| '.join(params) if len(params) > 0 else '' code = f''' -

Prompt: {prompt}

-

Negative: {negative}

-

Parameters: {params}

+

Prompt: {html.escape(prompt)}

+

Negative: {html.escape(negative)}

+

Parameters: {html.escape(params)}

''' return code @@ -134,6 +134,17 @@ def save_files(js_data, images, html_info, index): shutil.copy(fullfn, destination) shared.log.info(f'Copying image: file="{fullfn}" folder="{destination}"') tgt_filename = os.path.join(destination, os.path.basename(fullfn)) + if shared.opts.save_txt: + try: + from PIL import Image + image = Image.open(fullfn) + info, _ = images.read_info_from_image(image) + filename_txt = f"{os.path.splitext(tgt_filename)[0]}.txt" + with open(filename_txt, "w", encoding="utf8") as file: + file.write(f"{info}\n") + shared.log.debug(f'Saving: text="{filename_txt}"') + except Exception as e: + shared.log.warning(f'Image description save failed: {filename_txt} {e}') modules.script_callbacks.image_save_btn_callback(tgt_filename) else: image = image_from_url_text(filedata) diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 9727b8752..f30dba2cc 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -15,7 +15,7 @@ from collections import OrderedDict import gradio as gr from PIL import Image from starlette.responses import FileResponse, JSONResponse -from modules import paths, shared, scripts, modelloader +from modules import paths, shared, scripts, modelloader, errors from modules.ui_components import ToolButton import modules.ui_symbols as symbols @@ -270,7 +270,7 @@ class ExtraNetworksPage: self.html = f"
{subdirs_html}
{self.html}
" else: return '' - shared.log.debug(f"Extra networks: page='{self.name}' items={len(self.items)} subfolders={len(subdirs)} tab={tabname} folders={self.allowed_directories_for_previews()} list={self.list_time:.2f} desc={self.desc_time:.2f} info={self.info_time:.2f}") + shared.log.debug(f"Extra networks: page='{self.name}' items={len(self.items)} subfolders={len(subdirs)} tab={tabname} folders={self.allowed_directories_for_previews()} list={self.list_time:.2f} desc={self.desc_time:.2f} info={self.info_time:.2f} workers={shared.max_workers}") if len(self.missing_thumbs) > 0: threading.Thread(target=self.create_thumb).start() return self.html @@ -463,6 +463,10 @@ def create_ui(container, button_parent, tabname, skip_indexing = False): ui.tabs = gr.Tabs(elem_id=tabname+"_extra_tabs") ui.button_details = gr.Button('Details', elem_id=tabname+"_extra_details_btn", visible=False) state = {} + if shared.cmd_opts.profile: + import cProfile + pr = cProfile.Profile() + pr.enable() def get_item(state, params = None): if params is not None and type(params) == dict: @@ -567,6 +571,10 @@ def create_ui(container, button_parent, tabname, skip_indexing = False): page_html = gr.HTML(page.html, elem_id=f'{tabname}{page.name}_extra_page', elem_classes="extra-networks-page") ui.pages.append(page_html) tab.select(ui_tab_change, _js="getENActivePage", inputs=[ui.button_details], outputs=[ui.button_scan, ui.button_save, ui.button_model]) + if shared.cmd_opts.profile: + errors.profile(pr, 'ExtraNetworks') + pr.disable() + # ui.tabs.change(fn=ui_tab_change, inputs=[], outputs=[ui.button_scan, ui.button_save]) def fn_save_img(image): diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index 01902b376..692acdc18 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -64,7 +64,7 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): return record def list_items(self): - with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: future_items = {executor.submit(self.create_item, cp): cp for cp in list(sd_models.checkpoints_list.copy())} for future in concurrent.futures.as_completed(future_items): item = future.result() diff --git a/modules/ui_extra_networks_styles.py b/modules/ui_extra_networks_styles.py index 12f5f0a53..8dcccf96e 100644 --- a/modules/ui_extra_networks_styles.py +++ b/modules/ui_extra_networks_styles.py @@ -95,7 +95,7 @@ class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage): return item def list_items(self): - with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: future_items = {executor.submit(self.create_item, style): style for style in list(shared.prompt_styles.styles)} for future in concurrent.futures.as_completed(future_items): item = future.result() diff --git a/modules/ui_extra_networks_textual_inversion.py b/modules/ui_extra_networks_textual_inversion.py index f67031029..d94ce39dc 100644 --- a/modules/ui_extra_networks_textual_inversion.py +++ b/modules/ui_extra_networks_textual_inversion.py @@ -68,7 +68,7 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage): self.embeddings = [] self.embeddings = sorted(self.embeddings, key=lambda emb: emb.filename) - with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: future_items = {executor.submit(self.create_item, net): net for net in self.embeddings} for future in concurrent.futures.as_completed(future_items): item = future.result() diff --git a/modules/ui_models.py b/modules/ui_models.py index be338c23d..ae4baab15 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -81,7 +81,7 @@ def create_ui(): custom_name = gr.Textbox(label="New model name") with FormRow(): merge_mode = gr.Dropdown(choices=merge_methods.__all__, value="weighted_sum", label="Interpolation Method") - merge_mode_docs = gr.HTML(value=getattr(merge_methods, "weighted_sum").__doc__.replace("\n", "
")) + merge_mode_docs = gr.HTML(value=getattr(merge_methods, "weighted_sum", "").__doc__.replace("\n", "
")) with FormRow(): primary_model_name = gr.Dropdown(sd_model_choices(), label="Primary model", value="None") create_refresh_button(primary_model_name, sd_models.list_models, lambda: {"choices": sd_model_choices()}, "refresh_checkpoint_A") @@ -641,7 +641,7 @@ def create_ui(): civit_update_download_btn = gr.Button(value="Download", variant='primary', visible=False) class CivitModel: - def __init__(self, name, fn, sha = None, meta = {}): + def __init__(self, name, fn, sha = None, meta = {}): # noqa: B006 self.name = name self.id = meta.get('id', 0) self.fn = fn @@ -701,7 +701,7 @@ def create_ui(): model.latest_name = f.get('name', '') if model.vername == model.latest: model.status = 'Latest' - elif any(map(lambda v: v in model.latest_hashes, all_hashes)): + elif any(map(lambda v: v in model.latest_hashes, all_hashes)): # pylint: disable=cell-var-from-loop # noqa: C417 model.status = 'Downloaded' else: model.status = 'Available' @@ -716,7 +716,7 @@ def create_ui(): nonlocal selected_model, update_data try: selected_model = [m for m in update_data if m.fn == in_data[evt.index[0]][1]][0] - except: + except Exception: selected_model = None if selected_model is None or selected_model.url is None or selected_model.status != 'Available': return [gr.update(value='Model update not available'), gr.update(visible=False)] diff --git a/repositories/ldm/models/diffusion/ddpm.py b/repositories/ldm/models/diffusion/ddpm.py index 3350c032f..99879b224 100644 --- a/repositories/ldm/models/diffusion/ddpm.py +++ b/repositories/ldm/models/diffusion/ddpm.py @@ -81,7 +81,7 @@ class DDPM(pl.LightningModule): super().__init__() assert parameterization in ["eps", "x0", "v"], 'currently only supporting "eps" and "x0" and "v"' self.parameterization = parameterization - print(f"{self.__class__.__name__}: Running in {self.parameterization}-prediction mode") + print(f"{self.__class__.__name__}: mode={self.parameterization}") self.cond_stage_model = None self.clip_denoised = clip_denoised self.log_every_t = log_every_t diff --git a/repositories/ldm/modules/encoders/modules.py b/repositories/ldm/modules/encoders/modules.py index 523a7d853..8208d9471 100644 --- a/repositories/ldm/modules/encoders/modules.py +++ b/repositories/ldm/modules/encoders/modules.py @@ -301,8 +301,8 @@ class FrozenCLIPT5Encoder(AbstractEncoder): super().__init__() self.clip_encoder = FrozenCLIPEmbedder(clip_version, device, max_length=clip_max_length) self.t5_encoder = FrozenT5Embedder(t5_version, device, max_length=t5_max_length) - print(f"{self.clip_encoder.__class__.__name__} has {count_params(self.clip_encoder) * 1.e-6:.2f} M parameters, " - f"{self.t5_encoder.__class__.__name__} comes with {count_params(self.t5_encoder) * 1.e-6:.2f} M params.") + print(f"{self.clip_encoder.__class__.__name__} params={count_params(self.clip_encoder) * 1.e-6:.2f} M " + f"{self.t5_encoder.__class__.__name__} params={count_params(self.t5_encoder) * 1.e-6:.2f} M") def encode(self, text): return self(text) diff --git a/repositories/ldm/util.py b/repositories/ldm/util.py index 9ede259d5..bafa440d0 100644 --- a/repositories/ldm/util.py +++ b/repositories/ldm/util.py @@ -75,7 +75,7 @@ def mean_flat(tensor): def count_params(model, verbose=False): total_params = sum(p.numel() for p in model.parameters()) if verbose: - print(f"{model.__class__.__name__} has {total_params*1.e-6:.2f} M params.") + print(f"{model.__class__.__name__} params={total_params*1.e-6:.2f}M") return total_params diff --git a/requirements.txt b/requirements.txt index bbe4874e4..bf54222d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,7 +51,7 @@ requests==2.31.0 tqdm==4.66.1 accelerate==0.24.1 opencv-python-headless==4.7.0.72 -diffusers==0.23.1 +diffusers==0.24.0 einops==0.4.1 gradio==3.43.2 huggingface_hub==0.19.4 @@ -61,6 +61,7 @@ numba==0.57.1 pandas==1.5.3 protobuf==3.20.3 pytorch_lightning==1.9.4 +tokenizers==0.15.0 transformers==4.35.2 tomesd==0.1.3 urllib3==1.26.15 @@ -68,3 +69,4 @@ Pillow==10.1.0 timm==0.9.7 pydantic==1.10.13 typing-extensions==4.8.0 +peft diff --git a/scripts/animatediff.py b/scripts/animatediff.py new file mode 100644 index 000000000..4b0ef5d9c --- /dev/null +++ b/scripts/animatediff.py @@ -0,0 +1,163 @@ +""" +Lightweight AnimateDiff implementation in Diffusers +Docs: +TODO: +- SDXL +- Custom models +- Custom LORAs +- Enable second pass +- TemporalDiff: https://huggingface.co/CiaraRowles/TemporalDiff/tree/main +- AnimateFace: https://huggingface.co/nlper2022/animatediff_face_512/tree/main +""" + +import gradio as gr +import diffusers +from modules import scripts, processing, shared, devices, sd_models + + +# config +ADAPTERS = { + 'None': None, + 'Motion 1.4': 'guoyww/animatediff-motion-adapter-v1-4', + 'Motion 1.5 v1': 'guoyww/animatediff-motion-adapter-v1-5', + 'Motion 1.5 v2' :'guoyww/animatediff-motion-adapter-v1-5-2', + # 'Motion SD-XL Beta v1' :'vladmandic/animatediff-sdxl', + 'TemporalDiff': 'vladmandic/temporaldiff', + 'AnimateFace': 'vladmandic/animateface', +} +LORAS = { + 'None': None, + 'Zoom-in': 'guoyww/animatediff-motion-lora-zoom-in', + 'Zoom-out': 'guoyww/animatediff-motion-lora-zoom-out', + 'Pan-left': 'guoyww/animatediff-motion-lora-pan-left', + 'Pan-right': 'guoyww/animatediff-motion-lora-pan-right', + 'Tilt-up': 'guoyww/animatediff-motion-lora-tilt-up', + 'Tilt-down': 'guoyww/animatediff-motion-lora-tilt-down', + 'Roll-left': 'guoyww/animatediff-motion-lora-rolling-anticlockwise', + 'Roll-right': 'guoyww/animatediff-motion-lora-rolling-clockwise', +} + +# state +motion_adapter = None # instance of diffusers.MotionAdapter +loaded_adapter = None # name of loaded adapter +orig_pipe = None # original sd_model pipeline + + +def set_adapter(adapter_name: str = 'None'): + if shared.sd_model is None: + return + if shared.backend != shared.Backend.DIFFUSERS: + shared.log.warning('AnimateDiff: not in diffusers mode') + return + global motion_adapter, loaded_adapter, orig_pipe # pylint: disable=global-statement + # adapter_name = name if name is not None and isinstance(name, str) else loaded_adapter + if adapter_name is None or adapter_name == 'None' or shared.sd_model is None: + motion_adapter = None + loaded_adapter = None + if orig_pipe is not None: + shared.log.debug(f'AnimateDiff restore pipeline: adapter="{loaded_adapter}"') + shared.sd_model = orig_pipe + orig_pipe = None + return + if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl': + shared.log.warning(f'AnimateDiff: unsupported model type: {shared.sd_model.__class__.__name__}') + return + if motion_adapter is not None and loaded_adapter == adapter_name and shared.sd_model.__class__.__name__ == 'AnimateDiffPipeline': + shared.log.debug(f'AnimateDiff cache: adapter="{adapter_name}"') + return + if getattr(shared.sd_model, 'image_encoder', None) is not None: + shared.log.debug('AnimateDiff: unloading IP adapter') + # shared.sd_model.image_encoder = None + shared.sd_model.unet.set_default_attn_processor() + shared.sd_model.unet.config.encoder_hid_dim_type = None + try: + shared.log.info(f'AnimateDiff load: adapter="{adapter_name}"') + motion_adapter = None + motion_adapter = diffusers.MotionAdapter.from_pretrained(adapter_name, cache_dir=shared.opts.diffusers_dir, torch_dtype=devices.dtype, low_cpu_mem_usage=False, device_map=None) + motion_adapter.to(shared.device) + sd_models.set_diffuser_options(motion_adapter, vae=None, op='adapter') + loaded_adapter = adapter_name + + new_pipe = diffusers.AnimateDiffPipeline( + vae=shared.sd_model.vae, + text_encoder=shared.sd_model.text_encoder, + tokenizer=shared.sd_model.tokenizer, + unet=shared.sd_model.unet, + scheduler=shared.sd_model.scheduler, + motion_adapter=motion_adapter, + ) + orig_pipe = shared.sd_model + new_pipe.sd_checkpoint_info = shared.sd_model.sd_checkpoint_info + new_pipe.sd_model_hash = shared.sd_model.sd_model_hash + new_pipe.sd_model_checkpoint = shared.sd_model.sd_checkpoint_info.filename + new_pipe.is_sdxl = False + new_pipe.is_sd2 = False + new_pipe.is_sd1 = True + shared.sd_model = new_pipe + shared.sd_model.to(shared.device) + sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model') + shared.log.debug(f'AnimateDiff create pipeline: adapter="{loaded_adapter}"') + except Exception as e: + motion_adapter = None + loaded_adapter = None + shared.log.error(f'AnimateDiff load error: adapter="{adapter_name}" {e}') + + +class Script(scripts.Script): + def title(self): + return 'AnimateDiff' + + def show(self, _is_img2img): + return scripts.AlwaysVisible if shared.backend == shared.Backend.DIFFUSERS else False + + + def ui(self, _is_img2img): + def video_type_change(video_type): + return [ + gr.update(visible=video_type != 'None'), + gr.update(visible=video_type == 'GIF' or video_type == 'PNG'), + gr.update(visible=video_type == 'MP4'), + gr.update(visible=video_type == 'MP4'), + ] + + with gr.Accordion('AnimateDiff', open=False, elem_id='animatediff'): + with gr.Row(): + adapter_index = gr.Dropdown(label='Adapter', choices=list(ADAPTERS), value='None') + frames = gr.Slider(label='Frames', minimum=1, maximum=32, step=1, value=16) + with gr.Row(): + lora_index = gr.Dropdown(label='Lora', choices=list(LORAS), value='None') + strength = gr.Slider(label='Strength', minimum=0.0, maximum=2.0, step=0.05, value=1.0) + with gr.Row(): + latent_mode = gr.Checkbox(label='Latent mode', value=False) + with gr.Row(): + video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None') + duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False) + with gr.Row(): + gif_loop = gr.Checkbox(label='Loop', value=True, visible=False) + mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) + mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) + video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) + return [adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + + def process(self, p: processing.StableDiffusionProcessing, adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + adapter = ADAPTERS[adapter_index] + lora = LORAS[lora_index] + set_adapter(adapter) + if motion_adapter is None: + return + shared.log.debug(f'AnimateDiff: adapter="{adapter}" lora="{lora}" strength={strength} video={video_type}') + if lora is not None and lora != 'None': + shared.sd_model.load_lora_weights(lora, adapter_name=lora) + shared.sd_model.set_adapters([lora], adapter_weights=[strength]) + p.extra_generation_params['AnimateDiff Lora'] = f'{lora}:{strength}' + p.extra_generation_params['AnimateDiff'] = loaded_adapter + p.do_not_save_grid = True + p.task_args['num_frames'] = frames + p.task_args['num_inference_steps'] = p.steps + if not latent_mode: + p.task_args['output_type'] = 'np' + + def postprocess(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + from modules.images import save_video + if video_type != 'None': + save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) diff --git a/scripts/ipadapter.py b/scripts/ipadapter.py new file mode 100644 index 000000000..797ff2785 --- /dev/null +++ b/scripts/ipadapter.py @@ -0,0 +1,104 @@ +""" +Lightweight IP-Adapter applied to existing pipeline in Diffusers +- Downloads image_encoder or first usage (2.5GB) +- Introduced via: https://github.com/huggingface/diffusers/pull/5713 +- IP adapters: https://huggingface.co/h94/IP-Adapter +TODO: +- Additional IP addapters +- SD/SDXL autodetect +""" + +import gradio as gr +from modules import scripts, processing, shared, devices + + +image_encoder = None +loaded = None +ADAPTERS = [ + 'none', + 'models/ip-adapter_sd15', + 'models/ip-adapter_sd15_light', + # 'models/ip-adapter_sd15_vit-G', # RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x1024 and 1280x3072) + # 'models/ip-adapter-plus_sd15', # KeyError: 'proj.weight' + # 'models/ip-adapter-plus-face_sd15', # KeyError: 'proj.weight' + # 'models/ip-adapter-full-face_sd15', # KeyError: 'proj.weight' + 'sdxl_models/ip-adapter_sdxl', + # 'sdxl_models/ip-adapter_sdxl_vit-h', + # 'sdxl_models/ip-adapter-plus_sdxl_vit-h', + # 'sdxl_models/ip-adapter-plus-face_sdxl_vit-h', +] + + +class Script(scripts.Script): + def title(self): + return 'IP Adapter' + + def show(self, is_img2img): + return scripts.AlwaysVisible if shared.backend == shared.Backend.DIFFUSERS else False + + def ui(self, _is_img2img): + with gr.Accordion('IP Adapter', open=False, elem_id='ipadapter'): + with gr.Row(): + adapter = gr.Dropdown(label='Adapter', choices=ADAPTERS, value='none') + scale = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=0.5) + with gr.Row(): + image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512) + return [adapter, scale, image] + + def process(self, p: processing.StableDiffusionProcessing, adapter, scale, image): # pylint: disable=arguments-differ + import torch + from transformers import CLIPVisionModelWithProjection + + # init code + global loaded, image_encoder # pylint: disable=global-statement + if shared.sd_model is None: + return + if shared.backend != shared.Backend.DIFFUSERS: + shared.log.warning('IP adapter: not in diffusers mode') + return + if adapter == 'none': + if hasattr(shared.sd_model, 'set_ip_adapter_scale'): + shared.sd_model.set_ip_adapter_scale(0) + if loaded is not None: + shared.log.debug('IP adapter: unload attention processor') + shared.sd_model.unet.set_default_attn_processor() + shared.sd_model.unet.config.encoder_hid_dim_type = None + loaded = None + return + if image is None: + shared.log.error('IP adapter: no image') + return + if not hasattr(shared.sd_model, 'load_ip_adapter'): + shared.log.error(f'IP adapter: pipeline not supported: {shared.sd_model.__class__.__name__}') + return + if getattr(shared.sd_model, 'image_encoder', None) is None: + if shared.sd_model_type == 'sd': + subfolder = 'models/image_encoder' + elif shared.sd_model_type == 'sdxl': + subfolder = 'sdxl_models/image_encoder' + else: + shared.log.error(f'IP adapter: unsupported model type: {shared.sd_model_type}') + return + if image_encoder is None: + try: + image_encoder = CLIPVisionModelWithProjection.from_pretrained("h94/IP-Adapter", subfolder=subfolder, torch_dtype=torch.float16, cache_dir=shared.opts.diffusers_dir, use_safetensors=True).to(devices.device) + except Exception as e: + shared.log.error(f'IP adapter: failed to load image encoder: {e}') + return + + # main code + subfolder, model = adapter.split('/') + if model != loaded or getattr(shared.sd_model.unet.config, 'encoder_hid_dim_type', None) is None: + if loaded is not None: + shared.log.debug('IP adapter: reset attention processor') + shared.sd_model.unet.set_default_attn_processor() + loaded = None + shared.log.info(f'IP adapter load: adapter="{model}" scale={scale} image={image}') + shared.sd_model.image_encoder = image_encoder + shared.sd_model.load_ip_adapter("h94/IP-Adapter", subfolder=subfolder, weight_name=f'{model}.safetensors') + loaded = model + else: + shared.log.debug(f'IP adapter cache: adapter="{model}" scale={scale} image={image}') + shared.sd_model.set_ip_adapter_scale(scale) + p.task_args['ip_adapter_image'] = p.batch_size * [image] + p.extra_generation_params["IP Adapter"] = f'{adapter}:{scale}' diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py new file mode 100644 index 000000000..16077cc2c --- /dev/null +++ b/scripts/stablevideodiffusion.py @@ -0,0 +1,78 @@ +""" +Additional params for StableVideoDiffusion +""" +import torch +import gradio as gr +from modules import scripts, processing, shared, sd_models, images + + +class Script(scripts.Script): + def title(self): + return 'Stable Video Diffusion' + + def show(self, is_img2img): + return is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + + # return signature is array of gradio components + def ui(self, _is_img2img): + def video_type_change(video_type): + return [ + gr.update(visible=video_type != 'None'), + gr.update(visible=video_type == 'GIF' or video_type == 'PNG'), + gr.update(visible=video_type == 'MP4'), + gr.update(visible=video_type == 'MP4'), + ] + + with gr.Row(): + num_frames = gr.Slider(label='Frames', minimum=1, maximum=50, step=1, value=14) + min_guidance_scale = gr.Slider(label='Min guidance', minimum=0.0, maximum=10.0, step=0.1, value=1.0) + max_guidance_scale = gr.Slider(label='Max guidance', minimum=0.0, maximum=10.0, step=0.1, value=3.0) + with gr.Row(): + decode_chunk_size = gr.Slider(label='Decode chunks', minimum=1, maximum=25, step=1, value=6) + motion_bucket_id = gr.Slider(label='Motion level', minimum=0, maximum=1, step=0.05, value=0.5) + noise_aug_strength = gr.Slider(label='Noise strength', minimum=0.0, maximum=1.0, step=0.01, value=0.1) + with gr.Row(): + override_resolution = gr.Checkbox(label='Override resolution', value=True) + with gr.Row(): + video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None') + duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False) + with gr.Row(): + gif_loop = gr.Checkbox(label='Loop', value=True, visible=False) + mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) + mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) + video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) + return [num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + + def run(self, p: processing.StableDiffusionProcessing, num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + if shared.sd_model is None or shared.sd_model.__class__.__name__ != 'StableVideoDiffusionPipeline': + return None + if hasattr(p, 'init_images') and len(p.init_images) > 0: + if override_resolution: + p.width = 1024 + p.height = 576 + p.task_args['image'] = images.resize_image(resize_mode=2, im=p.init_images[0], width=p.width, height=p.height, upscaler_name=None, output_type='pil') + else: + p.task_args['image'] = p.init_images[0] + p.ops.append('svd') + p.do_not_save_grid = True + p.sampler_name = 'Default' # svd does not support non-default sampler + p.task_args['generator'] = torch.manual_seed(p.seed) # svd does not support gpu based generator + p.task_args['width'] = p.width + p.task_args['height'] = p.height + p.task_args['num_frames'] = num_frames + p.task_args['decode_chunk_size'] = decode_chunk_size + p.task_args['motion_bucket_id'] = round(255 * motion_bucket_id) + p.task_args['noise_aug_strength'] = noise_aug_strength + p.task_args['num_inference_steps'] = p.steps + p.task_args['min_guidance_scale'] = min_guidance_scale + p.task_args['max_guidance_scale'] = max_guidance_scale + p.task_args['output_type'] = 'np' + shared.log.debug(f'StableVideo: args={p.task_args}') + shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) + processed = processing.process_images(p) + if video_type != 'None': + images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) + return processed + else: + shared.log.error('StableVideo: no init_images') + return None diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index ad8cb65af..15d11e555 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -257,6 +257,12 @@ axis_options = [ AxisOption("[Refiner] Model", str, apply_refiner, fmt=format_value, cost=1.0, choices=lambda: ['None'] + sorted(sd_models.checkpoints_list)), AxisOption("[Refiner] Refiner start", float, apply_field("refiner_start")), AxisOption("[Refiner] Refiner steps", float, apply_field("refiner_steps")), + AxisOption("[HDR] Clamp boundary", float, apply_field("hdr_boundary")), + AxisOption("[HDR] Clamp threshold", float, apply_field("hdr_threshold")), + AxisOption("[HDR] Center channel shift", float, apply_field("hdr_channel_shift")), + AxisOption("[HDR] Center full shift", float, apply_field("hdr_full_shift")), + AxisOption("[HDR] Maximize center shift", float, apply_field("hdr_max_center")), + AxisOption("[HDR] Maximize boundary", float, apply_field("hdr_max_boundry")), AxisOption("[ToMe] Token merging ratio (txt2img)", float, apply_override('token_merging_ratio')), AxisOption("[ToMe] Token merging ratio (hires)", float, apply_override('token_merging_ratio_hr')), AxisOption("[FreeU] 1st stage backbone factor", float, apply_setting('freeu_b1')), diff --git a/webui.py b/webui.py index 46e3d7156..d352bed72 100644 --- a/webui.py +++ b/webui.py @@ -47,8 +47,6 @@ state = shared.state backend = shared.backend if not modules.loader.initialized: timer.startup.record("libraries") - log.setLevel(logging.DEBUG if cmd_opts.debug else logging.INFO) - logging.disable(logging.NOTSET if cmd_opts.debug else logging.DEBUG) if cmd_opts.server_name: server_name = cmd_opts.server_name else: @@ -212,7 +210,6 @@ def async_policy(): def start_common(): log.debug('Entering start sequence') - logging.disable(logging.NOTSET if cmd_opts.debug else logging.DEBUG) if shared.cmd_opts.data_dir is not None and len(shared.cmd_opts.data_dir) > 0: log.info(f'Using data path: {shared.cmd_opts.data_dir}') if shared.cmd_opts.models_dir is not None and len(shared.cmd_opts.models_dir) > 0 and shared.cmd_opts.models_dir != 'models': @@ -314,6 +311,9 @@ def webui(restart=False): modules.sd_models.write_metadata() load_model() shared.opts.save(shared.config_filename) + if cmd_opts.profile: + for k, v in modules.script_callbacks.callback_map.items(): + shared.log.debug(f'Registered callbacks: {k}={len(v)} {[c.script for c in v]}') log.info(f"Startup time: {timer.startup.summary()}") debug = log.info if os.environ.get('SD_SCRIPT_DEBUG', None) is not None else lambda *args, **kwargs: None debug('Loaded scripts:') diff --git a/wiki b/wiki index 6f0a39eda..931082304 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 6f0a39edad41d6d13b2822c7f93fab22d706c693 +Subproject commit 931082304da70e7683d993e28a1765c8cb6844c1