diff --git a/.eslintrc.json b/.eslintrc.json index 32283e620..81c196021 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -116,7 +116,8 @@ "idbDel": "readonly", "idbAdd": "readonly", "initChangelog": "readonly", - "sendNotification": "readonly" + "sendNotification": "readonly", + "monitorConnection": "readonly" }, "ignorePatterns": [ "node_modules", diff --git a/.gitignore b/.gitignore index 9a3415c86..9bc6d5a24 100644 --- a/.gitignore +++ b/.gitignore @@ -46,14 +46,6 @@ tunableop_results*.csv !package.json !requirements.txt -# pyinstaller -*.spec -build/ -dist/ - -# dynamically generated -/repositories/deepseek-vl2/ - # all dynamic stuff /extensions/**/* /outputs/**/* diff --git a/.gitmodules b/.gitmodules index 0ead18186..41ed87300 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,10 +14,6 @@ path = extensions-builtin/stable-diffusion-webui-rembg url = https://github.com/vladmandic/sd-extension-rembg ignore = dirty -[submodule "extensions-builtin/sd-webui-agent-scheduler"] - path = extensions-builtin/sd-webui-agent-scheduler - url = https://github.com/ArtVentureX/sd-webui-agent-scheduler - ignore = dirty [submodule "extensions-builtin/sdnext-modernui"] path = extensions-builtin/sdnext-modernui url = https://github.com/BinaryQuantumSoul/sdnext-modernui diff --git a/.pylintrc b/.pylintrc index 5bd9493dd..5dc22b99c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -29,6 +29,7 @@ ignore-paths=/usr/lib/.*$, modules/postprocess/aurasr_arch.py, modules/prompt_parser_xhinker.py, modules/ras, + modules/seedvr, modules/rife, modules/schedulers, modules/taesd, @@ -42,8 +43,10 @@ ignore-paths=/usr/lib/.*$, pipelines/meissonic, pipelines/omnigen2, pipelines/segmoe, + pipelines/xomni, scripts/consistory, scripts/ctrlx, + scripts/daam, scripts/demofusion, scripts/freescale, scripts/infiniteyou, @@ -73,7 +76,6 @@ persistent=yes py-version=3.9 recursive=no source-roots= -suggestion-mode=yes unsafe-load-any-extension=no [BASIC] diff --git a/.ruff.toml b/.ruff.toml index 74b734878..de4e0f7a5 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -12,6 +12,7 @@ exclude = [ "modules/pag", "modules/schedulers", "modules/teacache", + "modules/seedvr", "modules/control/proc", "modules/control/units", @@ -22,8 +23,10 @@ exclude = [ "pipelines/omnigen2", "pipelines/hdm", "pipelines/segmoe", + "pipelines/xomni", "scripts/lbm", + "scripts/daam", "scripts/xadapter", "scripts/pulid", "scripts/instantir", diff --git a/CHANGELOG.md b/CHANGELOG.md index a14c843c2..a175bfdb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,174 @@ # Change Log for SD.Next +## Update for 2025-10-17 + +### Highlights for 2025-10-17 + +It's been a month since the last release and number of changes is yet again massive with over 300 commits! +Highlight are: +- **Torch**: ROCm on Windows for AMD GPUs + if you have a compatible GPU, performance gains are significant! +- **Models**: + a lot of new stuff with **Qwen-Image-Edit** including multi-image edits and distilled variants, + new **Flux**, **WAN**, **LTX**, **HiDream** variants, expanded **Nunchaku** support and new SOTA upscaler with **SeedVR2** + plus improved video support in general, including new methods of video encoding +- **Quantization**: + new **SVD**-style quantization using SDNQ offers almost zero-loss even with **4bit** quantization + and now you can also test your favorite quantization on-the-fly and then save/load model for future use +- Other: support for **Huggingface** mirrors, changes to installer to prevent unwanted `torch-cpu` operations, improved previews, etc. + +### Details for 2025-10-17 + +- **Models** + - [WAN 2.2 14B VACE](https://huggingface.co/alibaba-pai/Wan2.2-VACE-Fun-A14B) + available for *text-to-image* and *text-to-video* and *image-to-video* workflows + - [Qwen Image Edit 2509](https://huggingface.co/Qwen/Qwen-Image-Edit-2509) and [Nunchaku Qwen Image Edit 2509](https://huggingface.co/nunchaku-tech/nunchaku-qwen-image-edit-2509) + updated version of Qwen Image Edit with improved image consistency + - [Qwen Image Pruning](https://huggingface.co/OPPOer/Qwen-Image-Pruning) and [Qwen Image Edit Pruning](https://huggingface.co/OPPOer/Qwen-Image-Edit-Pruning) + pruned versions of Qwen with 13B params instead of 20B, with some quality tradeoff + - [Tencent FLUX.1 Dev SRPO](https://huggingface.co/tencent/SRPO) + SRPO is trained by Tencent with specific technique: directly aligning the full diffusion trajectory with fine-grained human preference + - [Nunchaku SDXL](https://huggingface.co/nunchaku-tech/nunchaku-sdxl) and [Nunchaku SDXL Turbo](https://huggingface.co/nunchaku-tech/nunchaku-sdxl-turbo) + impact of nunchaku engine on unet-based model such as sdxl is much less than on a dit-based models, but its still significantly faster than baseline + note that nunchaku optimized and pre-quantized unet is replacement for base unet, so its only applicable to base models, not any of fine-tunes + *how to use*: enable nunchaku in settings -> quantization and then load either sdxl-base or sdxl-base-turbo reference models + - [HiDream E1.1](https://huggingface.co/HiDream-ai/HiDream-E1-1) + updated version of HiDream-E1 image editing model + - [LTXVideo 0.9.8](https://huggingface.co/Lightricks/LTX-Video-0.9.8-13B-distilled) + updated version of LTXVideo t2v/i2iv model + - [SeedVR2](https://iceclear.github.io/projects/seedvr/) + originally designed for video restoration, seedvr works great for image detailing and upscaling! + available in 3B, 7B and 7B-sharp variants, use as any other upscaler! + note: seedvr is a very large model (6.4GB and 16GB respectively) and not designed for lower-end hardware, quantization is highly recommended + note: seedvr is highly sensitive to its cfg scale, set in *settings -> postprocessing* + lower values will result in smoother output while higher values add details + - [X-Omni SFT](https://x-omni-team.github.io/) + *experimental*: X-omni is a transformer-only discrete auto-regressive image generative model trained with reinforcement learning +- **Features** + - **Model save**: ability to save currently loaded model as a new standalone model + why? SD.Next always prefers to start with full model and quantize on-demand during load + however, when you find your exact preferred quantization settings that work well for you, + saving such model as a new model allows for faster loads and reduced disk space usage + so its best of both worlds: you can experiment and test different quantization methods and once you find the one that works for you, save it as a new model + saved models appear in network tab as normal models and can be loaded as such + available in *models* tab + - [Qwen Image-Edit](https://huggingface.co/Qwen/Qwen-Image-Edit-2509) multi-image editing + requires qwen-image-edit-2509 or its variant as multi-image edits are not available in original qwen-image + in ui control tab: inputs -> separate init image + add image for *input media* and *control media* + can be + - [Cache-DiT](https://github.com/vipshop/cache-dit) + cache-dit is a unified, flexible and training-free cache acceleration framework + compatible with many dit-based models such as FLUX.1, Qwen, HunyuanImage, Wan2.2, Chroma, etc. + enable in *settings -> pipeline modifiers -> cache-dit* + - [Nunchaku Flux.1 PulID](https://nunchaku.tech/docs/nunchaku/python_api/nunchaku.pipeline.pipeline_flux_pulid.html) + automatically enabled if loaded model is FLUX.1 with Nunchaku engine enabled and when PulID script is enabled + - **Huggingface mirror** in *settings -> huggingface* + if you're working from location with limited access to huggingface, you can now specify a mirror site + for example enter, `https://hf-mirror.com` +- **Compute** + - **ROCm** for Windows + support for both official torch preview release of `torch-rocm` for windows and **TheRock** unofficial `torch-rocm` builds for windows + note that rocm for windows is still in preview and has limited gpu support, please check rocm docs for details + - **DirectML** warn as *end-of-life* + `torch-directml` received no updates in over 1 year and its currently superseded by `rocm` or `zluda` + - command line params `--use-zluda` and `--use-rocm` will attempt desired operation or fail if not possible + previously sdnext was performing a fallback to `torch-cpu` which is not desired + - **installer** if `--use-cuda` or `--use-rocm` are specified and `torch-cpu` is installed, installer will attempt to reinstall correct torch package + - **installer** warn if *cuda* or *rocm* are available and `torch-cpu` is installed + - support for `torch==2.10-nightly` with `cuda==13.0` +- **Extensions** + - [Agent-Scheduler](https://github.com/SipherAGI/sd-webui-agent-scheduler) + was a high-value built-in extension, but it has not been maintained for 1.5 years + it also does not work with control and video tabs which are the core of sdnext nowadays + so it has been removed from built-in extensions: manual installation is still possible + - [DAAM: Diffusion Attentive Attribution Maps](https://github.com/castorini/daam) + create heatmap visualizations of which parts of the prompt influenced which parts of the image + available in scripts for sdxl text-to-image workflows +- **Offloading** + - improve offloading for pipelines with multiple stages such as *wan-2.2-14b* + - add timers to measure onload/offload times during generate + - experimental offloading using `torch.streams` + enable in settings -> model offloading + - new feature to specify which models types not to offload + in *settings -> model offloading -> model types not to offload* +- **UI** + - **connection monitor** + main logo in top-left corner now indicates server connection status and hovering over it shows connection details + - separate guidance and detail sections + - networks ability to filter lora by base model version + - add interrogate button to input images + - disable spellchecks on all text inputs +- **SDNQ** + - add `SVDQuant` quantization method support + - make sdnq scales compatible with balanced offload + - add int8 `matmul` support for RDNA2 GPUs via triton + - improve int8 `matmul` performance on Intel GPUs +- **Other** + - server will note when restart is recommended due to package updates + - **interrupt** will now show last known preview image + *keep incomplete* setting is now *save interrupted* + - **logging** enable `debug`, `docs` and `api-docs` by default + - **logging** add detailed ram/vram utilization info to log + logging frequency can be specified using `--monitor x` command line param, where x is number of seconds + - **ipex** simplify internal implementation + - refactor to use new libraries + - styles and wildcards now use same seed as main generate for reproducible results + - **api** new endpoint POST `/sdapi/v1/civitai` to trigger civitai models metadata update + accepts optional `page` parameter to search specific networks page + - **reference models** additional example images, thanks @liutyi + - **reference models** add model size and release date, thanks @alerikaisattera + - **video** support for configurable multi-stage models such as WAN-2.2-14B + - **video** new LTX model selection + - replace `pynvml` with `nvidia-ml-py` for gpu monitoring + - update **loopback** script with radon seed option, thanks @rabanti + - **vae** slicing enable for *lowvram/medvram*, tiling for *lowvram*, both disabled otherwise + - **attention** remove split-attention and add explicitly attention slicing enable/disable option + enable in *settings -> compute settings* + can be combined with sdp, enabling may improve stability when used on iGPU or shared memory systems + - **nunchaku** update to `1.0.1` and enhance installer + - **xyz-grid** add guidance section + - **preview** implement configurable layers for WAN, Qwen, HV + - update swagger `/docs` endpoint style + - add `[epoch]` to filename template + - starting `[seq]` for filename template is now higher of largest previous sequence or number of files in folder +- **Video** + - use shared **T5** text encoder for video models when possible + - use shared **LLama** text encoder for video models when possible + - unified video save code across all video models + also avoids creation of temporary files for each frame unless user wants to save them + - unified prompt enhance code across all video models + - add job state tracking for video generation + - fix quantization not being applied on load for some models + - improve offloading for **ltx** and **wan** + - fix model selection in **ltx** tab +- **Experimental** + - `new` command line flag enables new `pydantic` and `albumentations` packages + - **modular pipelines**: enable in *settings -> model options* + only compatible with some pipelines, invalidates preview generation + - **modular guiders**: automatically used for compatible pipelines when *modular pipelines* is enabled + allows for using many different guidance methods: + *CFG, CFGZero, PAG, APG, SLG, SEG, TCFG, FDG* +- **Wiki** + - updates to *AMD-ROCm, ZLUDA, LoRA, DirectML, SDNQ, Quantization, Prompting, LoRA* pages + - new *Stability-Matrix* page +- **Fixes** + - **Microsoft Florence 2** both base and large variants + *note* this will trigger download of the new variant of the model, feel free to delete older variant in `huggingface` folder + - **MiaoshouAI PromptGen** 1.5/2.0 in both base and large variants + - fix prompt scheduling, thanks @nolbert82 + - ui: fix image metadata display when switching selected image in control tab + - framepack: add explicit hf-login before framepack load + - framepack: patch solver for unsupported gpus + - benchmark: remove forced sampler from system info benchmark + - xyz-grid: fix xyz grid with random seeds + - reference: fix download for sd15/sdxl reference models + - fix checks in init/mask image decode + - fix hf token with extra chars + - image viewer refocus on gallery after returning from full screen mode + - fix attention guidance metadata save/restore + - vae preview add explicity cuda.sync + ## Update for 2025-09-15 ### Highlights for 2025-09-15 @@ -8,7 +177,7 @@ **StandardUI** is still available and can be selected in settings, but ModernUI is now the default for new installs *What's else*? **Chroma** is in its final form, there are several new **Qwen-Image** variants and **Nunchaku** hit version 1.0! -Also, there are quite a few offloading improvements and many quality-of-life changes to UI and overal workflows +Also, there are quite a few offloading improvements and many quality-of-life changes to UI and overall workflows And check out new **history** tab in the right panel, it now shows visualization of entire processing timeline! ![Screenshot](https://github.com/user-attachments/assets/d6119a63-6ee5-4597-95f6-29ed0701d3b5) @@ -22,13 +191,13 @@ And check out new **history** tab in the right panel, it now shows visualization - **Qwen-Image** [InstantX ControlNet Union](https://huggingface.co/InstantX/Qwen-Image-ControlNet-Union) support *note* qwen-image is already a very large model and controlnet adds 3.5GB on top of that so quantization and offloading are highly recommended! - [Qwen-Lightning-Edit](https://huggingface.co/vladmandic/Qwen-Lightning-Edit) and [Qwen-Image-Distill](https://huggingface.co/SahilCarterr/Qwen-Image-Distill-Full) variants - - **Nuchaku** variants of [Qwen-Image-Lightning](https://huggingface.co/nunchaku-tech/nunchaku-qwen-image), [Qwen-Image-Edit](https://huggingface.co/nunchaku-tech/nunchaku-qwen-image-edit), [Nunchaku-Qwen-Image-Edit-Lightning](https://huggingface.co/nunchaku-tech/nunchaku-qwen-image-edit) + - **Nunchaku** variants of [Qwen-Image-Lightning](https://huggingface.co/nunchaku-tech/nunchaku-qwen-image), [Qwen-Image-Edit](https://huggingface.co/nunchaku-tech/nunchaku-qwen-image-edit), [Nunchaku-Qwen-Image-Edit-Lightning](https://huggingface.co/nunchaku-tech/nunchaku-qwen-image-edit) - **Nunchaku** variant of [Flux.1-Krea-Dev](https://huggingface.co/nunchaku-tech/nunchaku-flux.1-krea-dev) if you have a compatible nVidia GPU, Nunchaku is the fastest quantization & inference engine - [HunyuanDiT ControlNet](https://huggingface.co/Tencent-Hunyuan/HYDiT-ControlNet-v1.2) Canny, Depth, Pose - [KBlueLeaf/HDM-xut-340M-anime](https://huggingface.co/KBlueLeaf/HDM-xut-340M-anime) highly experimental: HDM *Home-made-Diffusion-Model* is a project to investigate specialized training recipe/scheme - for pretraining T2I model at home based on super-light architecture + for pre-training T2I model at home based on super-light architecture *requires*: generator=cpu, dtype=float16, offload=none, both positive and negative prompts are required and must be long & detailed - [Apple FastVLM](https://huggingface.co/apple/FastVLM-0.5B) in 0.5B, 1.5B and 7B variants available in captioning tab @@ -47,13 +216,13 @@ And check out new **history** tab in the right panel, it now shows visualization - additional artwork for reference models in networks, thanks @liutyi - improve ui hints display - restyled all toolbuttons to be modernui native - - reodered system settings + - reordered system settings - dynamic direction of dropdowns - improve process tab layout - improve detection of active tab - configurable horizontal vs vertical panel layout in settings -> user interface -> panel min width - *example*: if panel width is less than specified value, layout switches to verical + *example*: if panel width is less than specified value, layout switches to vertical - configurable grid images size in *settings -> user interface -> grid image size* - gallery now includes reference model images @@ -64,10 +233,10 @@ And check out new **history** tab in the right panel, it now shows visualization - improve offloading of models with multiple dits - improve offloading of models with impliciy vae processing - improve offloading of models with controlnet - - more aggressive offloading of controlnets with lowvram flag + - more aggressive offloading of controlnet with lowvram flag - **group** - new offloading method, using *type=leaf* works on a similar level as sequential offloading - and can present siginificant savings on low-vram gpus, but comes at the higher performace cost + and can present significant savings on low-vram gpus, but comes at the higher performance cost - **Quantization** - option to specify models types not to quantize: *settings -> quantization* allows for having quantization enabled, but skipping specific model types that do not need it @@ -82,6 +251,13 @@ And check out new **history** tab in the right panel, it now shows visualization *experimental*: requires new pydantic package which *may* break other things, to enable start sdnext with `--new` flag *note*: this is model quantization only, no support for tensorRT inference yet - **Other** + - **LoRA** allow specifying module to apply lora on + *example*: `` would apply lora *only* on unet regardless of lora content + this is particularly useful when you have multiple loras and you want to apply them on different parts of the model + *example*: `` and `` + *note*: `low` is shorthand for `module=transformer_2` and `high` is shortcut for `module=transformer` + - **Detailer** allow manually setting processing resolution + *note*: this does not impact the actual image resolution, only the resolution at which detailer internally operates - refactor reuse-seed and add functionality to all tabs - refactor modernui js codebase - move zluda flash attenion to *Triton Flash attention* option @@ -96,8 +272,6 @@ And check out new **history** tab in the right panel, it now shows visualization - add deprecation warning for `python==3.9` - allow setting denoise strength to 0 in control/img2img this allows to run workflows which only refine or detail existing image without changing it - - **Detailer** allow manually setting processing resolution - *note*: this does not impact the actual image resolution, only the resolution at which detailer internally operates - **Fixes** - normalize path hanlding when deleting images - unified compile upscalers diff --git a/README.md b/README.md index e37f77479..b81f1a8aa 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ ![Last update](https://img.shields.io/github/last-commit/vladmandic/sdnext?svg=true) ![License](https://img.shields.io/github/license/vladmandic/sdnext?svg=true) [![Discord](https://img.shields.io/discord/1101998836328697867?logo=Discord&svg=true)](https://discord.gg/VjvR2tabEX) +[![DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/vladmandic/sdnext) [![Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/vladmandic) [Docs](https://vladmandic.github.io/sdnext-docs/) | [Wiki](https://github.com/vladmandic/sdnext/wiki) | [Discord](https://discord.gg/VjvR2tabEX) | [Changelog](CHANGELOG.md) @@ -30,12 +31,12 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG ▹ **Standard | Modern** - Multiple [diffusion models](https://vladmandic.github.io/sdnext-docs/Model-Support/)! - Built-in Control for Text, Image, Batch and Video processing! -- Multiplatform! +- Multi-platform! ▹ **Windows | Linux | MacOS | nVidia CUDA | AMD ROCm | Intel Arc / IPEX XPU | DirectML | OpenVINO | ONNX+Olive | ZLUDA** -- Platform specific autodetection and tuning performed on install +- Platform specific auto-detection and tuning performed on install - Optimized processing with latest `torch` developments with built-in support for model compile and quantize Compile backends: *Triton | StableFast | DeepCache | OneDiff | TeaCache | etc.* - Quantization methods: *SDNQ | BitsAndBytes | Optimum-Quanto | TorchAO* + Quantization methods: *SDNQ | BitsAndBytes | Optimum-Quanto | TorchAO / LayerWise* - **Interrogate/Captioning** with 150+ **OpenCLiP** models and 20+ built-in **VLMs** - Built-in queue management - Built in installer with automatic updates and dependency management @@ -53,7 +54,7 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG screenshot-modernui-mobile -For screenshots and informations on other available themes, see [Themes](https://vladmandic.github.io/sdnext-docs/Themes/) +For screenshots and information on other available themes, see [Themes](https://vladmandic.github.io/sdnext-docs/Themes/)
@@ -74,7 +75,7 @@ SD.Next supports broad range of models: [supported models](https://vladmandic.gi - *ONNX/Olive* - *AMD* GPUs on Windows using **ZLUDA** libraries -Plus Docker container receipes for: [CUDA, ROCm, Intel IPEX and OpenVINO](https://vladmandic.github.io/sdnext-docs/Docker/) +Plus Docker container recipes for: [CUDA, ROCm, Intel IPEX and OpenVINO](https://vladmandic.github.io/sdnext-docs/Docker/) ## Getting started diff --git a/TODO.md b/TODO.md index dce3ad49e..5bdc45124 100644 --- a/TODO.md +++ b/TODO.md @@ -13,21 +13,15 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - Video Core: API - Video LTX: TeaCache and others, API, Conditioning preprocess Video: LTX API -### Blocked items - -- Upgrade: `albumentations` - - blocked by `insightface` - ### Under Consideration -- [X-Omni](https://github.com/X-Omni-Team/X-Omni/blob/main/README.md) +- [Inf-DiT](https://github.com/zai-org/Inf-DiT) - [DiffSynth Studio](https://github.com/modelscope/DiffSynth-Studio) - [IPAdapter negative guidance](https://github.com/huggingface/diffusers/discussions/7167) - [IPAdapter composition](https://huggingface.co/ostris/ip-composition-adapter) - [STG](https://github.com/huggingface/diffusers/blob/main/examples/community/README.md#spatiotemporal-skip-guidance) - [SmoothCache](https://github.com/huggingface/diffusers/issues/11135) - [MagCache](https://github.com/lllyasviel/FramePack/pull/673/files) -- [Nunchaku PulID](https://github.com/mit-han-lab/nunchaku/pull/274) - [Dream0 guidance](https://huggingface.co/ByteDance/DreamO) - [SUPIR upscaler](https://github.com/Fanghua-Yu/SUPIR) - [ByteDance OneReward](https://github.com/bytedance/OneReward) @@ -40,14 +34,19 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ### New models +- [Ovi](https://github.com/character-ai/Ovi) +- [Kandinsky 5](https://github.com/ai-forever/Kandinsky-5) +- [Bytedance Lynx](https://github.com/bytedance/lynx) - [HunyuanImage](https://huggingface.co/tencent/HunyuanImage-2.1) +- [Phantom HuMo](https://github.com/Phantom-video/Phantom) - [Lumina-DiMOO](https://huggingface.co/Alpha-VLLM/Lumina-DiMOO) -- [Wan2.2 Fun](https://huggingface.co/collections/alibaba-pai/wan22-fun-68958eabec343b948f1225c5) (includes VACE, Control, etc) +- [Wan2.2-Animate-14B](https://huggingface.co/Wan-AI/Wan2.2-Animate-14B) - [Magi](https://github.com/SandAI-org/MAGI-1)(https://github.com/huggingface/diffusers/pull/11713) - [SEVA](https://github.com/huggingface/diffusers/pull/11440) - [Ming](https://github.com/inclusionAI/Ming) - [Liquid](https://github.com/FoundationVision/Liquid) - [Step1X](https://github.com/stepfun-ai/Step1X-Edit) +- [LucyEdit](https://github.com/huggingface/diffusers/pull/12340) - [SD3 UltraEdit](https://github.com/HaozheZhao/UltraEdit) - [WAN2GP](https://github.com/deepbeepmeep/Wan2GP) - [SelfForcing](https://github.com/guandeh17/Self-Forcing) diff --git a/cli/localize.js b/cli/localize.js index da870d455..12d289867 100755 --- a/cli/localize.js +++ b/cli/localize.js @@ -7,9 +7,10 @@ const { GoogleGenerativeAI } = require('@google/generative-ai'); const api_key = process.env.GOOGLE_AI_API_KEY; const model = 'gemini-2.5-flash'; -const prompt = ` -// eslint-disable-next-line max-len -Translate attached JSON from English to {language} using following rules: fields id, label and reload should be preserved from original, field localized should be a translated version of field label and field hint should be translated in-place. if field is less than 3 characters, do not translate it and keep it as is. Every JSON entry should have id, label, localized, reload and hint fields. Output should be pure JSON without any additional text. To better match translation, context of the text is related to Stable Diffusion and topic of Generative AI.`; +const prompt = `Translate attached JSON from English to {language} using following rules: fields id, label and reload should be preserved from original, field localized should be a translated version of field label and field hint should be translated in-place. +if field is less than 3 characters, do not translate it and keep it as is. +Every JSON entry should have id, label, localized, reload and hint fields. +Output should be pure JSON without any additional text. To better match translation, context of the text is related to Stable Diffusion and topic of Generative AI.`; const languages = { hr: 'Croatian', de: 'German', diff --git a/configs/Dockerfile.cuda b/configs/Dockerfile.cuda index 4e5a496e1..b53a8872f 100644 --- a/configs/Dockerfile.cuda +++ b/configs/Dockerfile.cuda @@ -41,11 +41,11 @@ ENV SD_DOCKER=true # tcmalloc is not required but it is highly recommended ENV LD_PRELOAD=libtcmalloc.so.4 # sdnext will run all necessary pip install ops and then exit -RUN ["python", "/app/launch.py", "--debug", "--uv", "--use-cuda", "--log", "sdnext.log", "--test", "--optional"] +RUN ["python", "/app/launch.py", "--uv", "--use-cuda", "--log", "sdnext.log", "--test", "--optional"] # preinstall additional packages to avoid installation during runtime # actually run sdnext -CMD ["python", "launch.py", "--debug", "--skip-all", "--listen", "--quick", "--api-log", "--log", "sdnext.log"] +CMD ["python", "launch.py", "--listen", "--quick", "--log", "sdnext.log"] # expose port EXPOSE 7860 diff --git a/extensions-builtin/sd-extension-chainner b/extensions-builtin/sd-extension-chainner index 3e0108fed..c6dc85eb2 160000 --- a/extensions-builtin/sd-extension-chainner +++ b/extensions-builtin/sd-extension-chainner @@ -1 +1 @@ -Subproject commit 3e0108fedbec300f72c3ca6e06236419d45eb660 +Subproject commit c6dc85eb28a02bc7af268497b7a5a596770c5d7b diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index 615d2f810..19a1b1b72 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit 615d2f8103fb61779037bd81523906457fbf7277 +Subproject commit 19a1b1b722b768b821f67831a6d3ba83847bcb61 diff --git a/extensions-builtin/sd-webui-agent-scheduler b/extensions-builtin/sd-webui-agent-scheduler deleted file mode 160000 index a33753321..000000000 --- a/extensions-builtin/sd-webui-agent-scheduler +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a33753321b914c6122df96d1dc0b5117d38af680 diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 430c8140c..db79b18f6 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 430c8140c0cdda5e82a80f44ee6961e0d803e516 +Subproject commit db79b18f6f3f5a247e710b507d10cb39b01cc371 diff --git a/html/locale_en.json b/html/locale_en.json index 8113a811b..42d81e431 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -32,7 +32,6 @@ {"id":"","label":"","localized":"","reload":"","hint":"Sort by time, descending"} ], "main": [ - {"id":"","label":"SD.Next","localized":"","reload":"","hint":"SD.Next
All-in-one WebUI for AI generative image and video creation"}, {"id":"","label":"Prompt","localized":"","reload":"","hint":"Describe image you want to generate"}, {"id":"","label":"Start","localized":"","reload":"","hint":"Start"}, {"id":"","label":"End","localized":"","reload":"","hint":"End"}, @@ -127,7 +126,7 @@ {"id":"","label":"➠ Sketch","localized":"","reload":"","hint":"Transfer image to sketch interface"}, {"id":"","label":"➠ Composite","localized":"","reload":"","hint":"Transfer image to inpaint sketch interface"}, {"id":"","label":"➠ Process","localized":"","reload":"","hint":"Transfer image to process interface"}, - {"id":"","label":"➠ Control","localized":"","reload":"","hint":"Transfrer image to control interface"}, + {"id":"","label":"➠ Control","localized":"","reload":"","hint":"Transfer image to control interface"}, {"id":"","label":"➠ Caption","localized":"","reload":"","hint":"Transfer image to caption interface"} ], "generate": [ diff --git a/html/previews.json b/html/previews.json index 6f77db164..07af08547 100644 --- a/html/previews.json +++ b/html/previews.json @@ -1,6 +1,6 @@ { "sd-v21-512-ema": "models/Reference/stabilityai--stable-diffusion-2-1-base.jpg", - "stabilityai--stable-diffusion-xl-base-1.0": "models/Reference/sd_xl_base_1.0.jpg", + "stabilityai--stable-diffusion-xl-base-1.0": "models/Reference/stabilityai--stable-diffusion-xl-base-1.0.jpg", "stabilityai--stable-diffusion-3-medium-diffusers": "models/Reference/stabilityai--stable-diffusion-3.jpg", "stabilityai--stable-diffusion-3.5-medium": "models/Reference/stabilityai--stable-diffusion-3_5-medium.jpg", "stabilityai--stable-diffusion-3.5-large": "models/Reference/stabilityai--stable-diffusion-3_5-large.jpg", @@ -20,10 +20,11 @@ "HiDream-I1-Full": "models/Reference/HiDream-I1 Full", "lodestones--Chroma1-Base": "models/Reference/lodestones--Chroma-Base.jpg", "lodestones--Chroma1-HD": "models/Reference/lodestones--Chroma-HD.jpg", - "chroma-unlocked-v50": "models/Reference/lodestones Chroma Unlocked HD", - "chroma-unlocked-v50-annealed": "models/Reference/lodestones Chroma Unlocked HD", + "chroma-unlocked-v50": "models/Reference/lodestones--Chroma-detail.jpg", + "chroma-unlocked-v50-annealed": "models/Reference/lodestones--Chroma-annealed.jpg", "vladmandic--Qwen-Lightning": "models/Reference/Qwen-Lightning.jpg", "vladmandic--Qwen-Lightning-Edit": "models/Reference/Qwen-Lightning.jpg", - "Wan-AI--Wan2.2-T2V-A14B-Diffusers": "models/Reference/Wan2.2-T2V-A14B.jpg", - "Wan-AI--Wan2.1-T2V-14B-Diffusers": "models/Reference/Wan-AI--Wan2.1.jpg" + "Wan-AI--Wan2.2-T2V-A14B-Diffusers": "models/Reference/Wan-AI--Wan2.2-T2V-A14B-Diffusers.jpg", + "Wan-AI--Wan2.1-T2V-14B-Diffusers": "models/Reference/Wan-AI--Wan2.1-T2V-14B-Diffusers.jpg", + "linoyts--Wan2.2-VACE-Fun-14B-diffusers": "models/Reference/linoyts--Wan2.2-VACE-Fun-14B-diffusers.jpg" } diff --git a/html/reference.json b/html/reference.json index ee0ccf103..fced58542 100644 --- a/html/reference.json +++ b/html/reference.json @@ -1,26 +1,26 @@ { "Tempest-by-Vlad XL": { "path": "tempestByVlad_baseV01.safetensors@https://civitai.com/api/download/models/1301775", - "preview": "tempest-by-vlad-base.jpg", + "preview": "tempestByVlad_baseV01.jpg", "desc": "Flexible SDXL model with custom encoder and finetuned for larger landscape resolutions with high details and high contrast.", "extras": "" }, "Tempest-by-Vlad XL Hyper": { "path": "tempestByVlad_hyperV01.safetensors@https://civitai.com/api/download/models/1343512", - "preview": "tempest-by-vlad-hyper.jpg", + "preview": "tempestByVlad_hyperV01.jpg", "desc": "Custom distilled variant with goal to get as-normal-as-possible model that works with low steps and guidance-free", "extras": "" }, "Juggernaut XL XI": { "path": "juggernautXL_juggXIByRundiffusion.safetensors@https://civitai.com/api/download/models/782002", - "preview": "juggernautXL_v9Rundiffusionphoto2.jpg", + "preview": "juggernautXL_juggXIByRundiffusion.jpg", "desc": "Showcase finetuned model based on Stable diffusion XL", "extras": "sampler: DEIS, steps: 20, cfg_scale: 6.0" }, "Juggernaut XL XI Lightning": { "path": "juggernautXL_juggXILightningByRD.safetensors@https://civitai.com/api/download/models/920957", - "preview": "juggernautXL_v9Rdphoto2Lightning.jpg", + "preview": "juggernautXL_juggXILightningByRD.jpg", "desc": "Showcase finetuned model based on Stable diffusion XL", "extras": "sampler: DPM SDE, steps: 6, cfg_scale: 2.0" }, @@ -37,7 +37,9 @@ "path": "v1-5-pruned-fp16-emaonly.safetensors@https://huggingface.co/Aptronym/SDNext/resolve/main/Reference/v1-5-pruned-fp16-emaonly.safetensors?download=true", "preview": "v1-5-pruned-fp16-emaonly.jpg", "desc": "Stable Diffusion 1.5 is the base model all other 1.5 checkpoint were trained from. It's a latent text-to-image diffusion model capable of generating photo-realistic images given any text input. The Stable-Diffusion-v1-5 checkpoint was initialized with the weights of the Stable-Diffusion-v1-2 checkpoint and subsequently fine-tuned on 595k steps at resolution 512x512.", - "extras": "width: 512, height: 512, sampler: DEIS, steps: 20, cfg_scale: 6.0" + "extras": "width: 512, height: 512, sampler: DEIS, steps: 20, cfg_scale: 6.0", + "size": 2.28, + "date": "2022 October" }, "StabilityAI StableDiffusion 2.1": { "path": "huggingface/stabilityai/stable-diffusion-2-1-base", @@ -45,7 +47,9 @@ "skip": true, "variant": "fp16", "desc": "This stable-diffusion-2-1-base model fine-tunes stable-diffusion-2-base (512-base-ema.ckpt) with 220k extra steps taken", - "extras": "width: 512, height: 512, sampler: DEIS, steps: 20, cfg_scale: 6.0" + "extras": "width: 512, height: 512, sampler: DEIS, steps: 20, cfg_scale: 6.0", + "size": 2.58, + "date": "2022 December" }, "StabilityAI StableDiffusion 2.1 V": { "path": "huggingface/stabilityai/stable-diffusion-2-1", @@ -55,11 +59,23 @@ "desc": "This stable-diffusion-2 model is resumed from stable-diffusion-2-base (512-base-ema.ckpt) and trained for 150k steps using a v-objective on the same dataset. Resumed for another 140k steps on 768x768 images", "extras": "width: 768, height: 768, sampler: DEIS, steps: 20, cfg_scale: 6.0" }, - "StabilityAI StableDiffusion XL 1.0 Base": { - "path": "sd_xl_base_1.0.safetensors@https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0.safetensors?download=true", - "preview": "sd_xl_base_1.0.jpg", - "desc": "Stable Diffusion XL (SDXL) is the latest AI image generation model that is tailored towards more photorealistic outputs with more detailed imagery and composition compared to previous SD models, including SD 2.1. It can make realistic faces, legible text within the images, and better image composition, all while using shorter and simpler prompts at a greatly increased base resolution of 1024x1024. Just like its predecessors, SDXL has the ability to generate image variations using image-to-image prompting, inpainting (reimagining of the selected parts of an image), and outpainting (creating new parts that lie outside the image borders).", - "extras": "sampler: DEIS, steps: 20, cfg_scale: 6.0" + "StabilityAI StableDiffusion XL": { + "path": "stabilityai/stable-diffusion-xl-base-1.0", + "preview": "stabilityai--stable-diffusion-xl-base-1.0.jpg", + "desc": "Stable Diffusion XL (SDXL) is AI image generation model that is tailored towards more photorealistic outputs with more detailed imagery and composition compared to previous SD models, including SD 2.1. It can make realistic faces and better image composition, all while using shorter and simpler prompts at a greatly increased base resolution of 1024x1024. Just like its predecessors, SDXL has the ability to generate image variations using image-to-image prompting, inpainting (reimagining of the selected parts of an image), and outpainting (creating new parts that lie outside the image borders).", + "skip": true, + "variant": "fp16", + "extras": "", + "size": 6.94, + "date": "2023 July" + }, + "StabilityAI StableDiffusion XL Turbo": { + "path": "stabilityai/sdxl-turbo", + "preview": "stabilityai--sdxl-turbo.jpg", + "desc": "SDXL-Turbo is a fast generative text-to-image model that can synthesize photorealistic images from a text prompt in a 1-4 steps.", + "skip": true, + "variant": "fp16", + "extras": "steps: 4, cfg_scale: 0.0" }, "StabilityAI Stable Cascade": { "path": "huggingface/stabilityai/stable-cascade", @@ -67,7 +83,9 @@ "variant": "bf16", "desc": "Stable Cascade is a diffusion model built upon the Würstchen architecture and its main difference to other models like Stable Diffusion is that it is working at a much smaller latent space. Why is this important? The smaller the latent space, the faster you can run inference and the cheaper the training becomes. How small is the latent space? Stable Diffusion uses a compression factor of 8, resulting in a 1024x1024 image being encoded to 128x128. Stable Cascade achieves a compression factor of 42, meaning that it is possible to encode a 1024x1024 image to 24x24, while maintaining crisp reconstructions. The text-conditional model is then trained in the highly compressed latent space. Previous versions of this architecture, achieved a 16x cost reduction over Stable Diffusion 1.5", "preview": "stabilityai--stable-cascade.jpg", - "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0" + "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0", + "size": 11.82, + "date": "2024 February" }, "StabilityAI Stable Cascade Lite": { "path": "huggingface/stabilityai/stable-cascade-lite", @@ -75,7 +93,9 @@ "variant": "bf16", "desc": "Stable Cascade is a diffusion model built upon the Würstchen architecture and its main difference to other models like Stable Diffusion is that it is working at a much smaller latent space. Why is this important? The smaller the latent space, the faster you can run inference and the cheaper the training becomes. How small is the latent space? Stable Diffusion uses a compression factor of 8, resulting in a 1024x1024 image being encoded to 128x128. Stable Cascade achieves a compression factor of 42, meaning that it is possible to encode a 1024x1024 image to 24x24, while maintaining crisp reconstructions. The text-conditional model is then trained in the highly compressed latent space. Previous versions of this architecture, achieved a 16x cost reduction over Stable Diffusion 1.5", "preview": "stabilityai--stable-cascade-lite.jpg", - "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0" + "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0", + "size": 4.97, + "date": "2024 February" }, "StabilityAI Stable Diffusion 3 Medium": { "path": "stabilityai/stable-diffusion-3-medium-diffusers", @@ -83,7 +103,9 @@ "variant": "fp16", "desc": "Stable Diffusion 3 Medium is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features greatly improved performance in image quality, typography, complex prompt understanding, and resource-efficiency", "preview": "stabilityai--stable-diffusion-3.jpg", - "extras": "sampler: Default, cfg_scale: 7.0" + "extras": "sampler: Default, cfg_scale: 7.0", + "size": 15.14, + "date": "2024 June" }, "StabilityAI Stable Diffusion 3.5 Medium": { "path": "stabilityai/stable-diffusion-3.5-medium", @@ -91,7 +113,9 @@ "variant": "fp16", "desc": "Stable Diffusion 3.5 Medium is a Multimodal Diffusion Transformer with improvements (MMDiT-X) text-to-image model that features improved performance in image quality, typography, complex prompt understanding, and resource-efficiency.", "preview": "stabilityai--stable-diffusion-3_5-medium.jpg", - "extras": "sampler: Default, cfg_scale: 7.0" + "extras": "sampler: Default, cfg_scale: 7.0", + "size": 15.89, + "date": "2024 October" }, "StabilityAI Stable Diffusion 3.5 Large": { "path": "stabilityai/stable-diffusion-3.5-large", @@ -99,7 +123,9 @@ "variant": "fp16", "desc": "Stable Diffusion 3.5 Large is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features improved performance in image quality, typography, complex prompt understanding, and resource-efficiency.", "preview": "stabilityai--stable-diffusion-3_5-large.jpg", - "extras": "sampler: Default, cfg_scale: 7.0" + "extras": "sampler: Default, cfg_scale: 7.0", + "size": 26.98, + "date": "2024 October" }, "StabilityAI Stable Diffusion 3.5 Turbo": { "path": "stabilityai/stable-diffusion-3.5-large-turbo", @@ -115,107 +141,177 @@ "preview": "black-forest-labs--FLUX.1-dev.jpg", "desc": "FLUX.1 models are based on a hybrid architecture of multimodal and parallel diffusion transformer blocks, scaled to 12B parameters and builing on flow matching", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 32.93, + "date": "2024 August" }, "Black Forest Labs FLUX.1 Schnell": { "path": "black-forest-labs/FLUX.1-schnell", "preview": "black-forest-labs--FLUX.1-schnell.jpg", "desc": "FLUX.1 models are based on a hybrid architecture of multimodal and parallel diffusion transformer blocks, scaled to 12B parameters and builing on flow matching. Trained using latent adversarial diffusion distillation, FLUX.1 [schnell] can generate high-quality images in only 1 to 4 steps", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 32.93, + "date": "2024 August" }, "Black Forest Labs FLUX.1 Kontext Dev": { "path": "black-forest-labs/FLUX.1-Kontext-dev", "preview": "black-forest-labs--FLUX.1-Kontext-dev.jpg", "desc": "FLUX.1 Kontext [dev] is a 12 billion parameter rectified flow transformer capable of editing images based on text instructions.", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 32.93, + "date": "2025 June" }, "Black Forest Labs FLUX.1 Krea Dev": { "path": "black-forest-labs/FLUX.1-Krea-dev", "preview": "black-forest-labs--FLUX.1-Krea-dev.jpg", "desc": "FLUX.1 Krea [dev] is a 12 billion parameter rectified flow transformer capable of generating images from text descriptions.", "skip": true, + "extras": "sampler: Default, cfg_scale: 4.5", + "size": 32.93, + "date": "2025 July" + }, + "Tencent FLUX.1 Dev SRPO": { + "path": "vladmandic/flux.1-dev-SRPO", + "preview": "vladmandic--flux.1-dev-SRPO.jpg", + "desc": "FLUX.1 Dev SRPO is Tencent trained with specific technique: Directly Aligning the Full Diffusion Trajectory with Fine-Grained Human Preference", + "skip": true, "extras": "sampler: Default, cfg_scale: 4.5" }, + "Qwen-Image": { + "path": "Qwen/Qwen-Image", + "preview": "Qwen--Qwen-Image.jpg", + "desc": "Qwen-Image, an image generation foundation model in the Qwen series that achieves significant advances in complex text rendering and precise image editing.", + "skip": true, + "extras": "", + "size": 56.1, + "date": "2025 August" + }, + "Qwen-Image-Edit": { + "path": "Qwen/Qwen-Image-Edit", + "preview": "Qwen--Qwen-Image-Edit.jpg", + "desc": "Qwen-Image-Edit, the image editing version of Qwen-Image. Built upon our 20B Qwen-Image model, Qwen-Image-Edit successfully extends Qwen-Image’s unique text rendering capabilities to image editing tasks, enabling precise text editing.", + "skip": true, + "extras": "", + "size": 56.1, + "date": "2025 August" + }, + "Qwen-Image-Edit-2509": { + "path": "Qwen/Qwen-Image-Edit-2509", + "preview": "Qwen--Qwen-Image-Edit-2509.jpg", + "desc": "Qwen-Image-Edit, the image editing version of Qwen-Image. Built upon our 20B Qwen-Image model, Qwen-Image-Edit successfully extends Qwen-Image’s unique text rendering capabilities to image editing tasks, enabling precise text editing.", + "skip": true, + "extras": "", + "size": 56.1, + "date": "2025 September" + }, + "Qwen-Image-Lightning": { + "path": "vladmandic/Qwen-Lightning", + "preview": "vladmandic--Qwen-Lightning.jpg", + "desc": "Qwen-Lightning is step-distilled from Qwen-Image to allow for generation in 8 steps.", + "skip": true, + "extras": "steps: 8", + "size": 56.1, + "date": "2025 August" + }, + "Qwen-Image-Distill": { + "path": "SahilCarterr/Qwen-Image-Distill-Full", + "preview": "SahilCarterr--Qwen-Image-Distill-Full.jpg", + "desc": "Qwen-Image-Distill is a distilled and accelerated version of Qwen-Image by DiffSynth-Studio.", + "skip": true, + "extras": "steps: 15", + "size": 56.1, + "date": "2025 August" + }, + "Qwen-Image-Lightning-Edit": { + "path": "vladmandic/Qwen-Lightning-Edit", + "preview": "vladmandic--Qwen-Lightning-Edit.jpg", + "desc": "Qwen-Lightning-Edit is step-distilled from Qwen-Image-Edit to allow for generation in 8 steps.", + "skip": true, + "extras": "steps: 8", + "size": 56.1, + "date": "2025 August" + }, + "Qwen-Image Pruning-12B": { + "path": "OPPOer/Qwen-Image-Pruning", + "subfolder": "Qwen-Image-12B-8steps", + "preview": "OPPOer--Qwen-Image-Pruning.jpg", + "desc": "This open-source project is based on Qwen-Image and has attempted model pruning, removing 20 layers while retaining the weights of 40 layers, resulting in a model size of 12B parameters.", + "skip": true, + "date": "2025 Ocotober" + }, + "Qwen-Image-Edit Pruning-13B": { + "path": "OPPOer/Qwen-Image-Edit-Pruning", + "subfolder": "Qwen-Image-Edit-13B-4steps", + "preview": "OPPOer--Qwen-Image-Edit-Pruning.jpg", + "desc": "This open-source project is based on Qwen-Image-Edit and has attempted model pruning, removing 20 layers while retaining the weights of 40 layers, resulting in a model size of 13.6B parameters.", + "skip": true, + "date": "2025 Ocotober" + }, + "Qwen-Image-Edit-2509 Pruning-13B": { + "path": "OPPOer/Qwen-Image-Edit-2509-Pruning", + "subfolder": "Qwen-Image-Edit-2509-13B-4steps", + "preview": "OPPOer--Qwen-Image-Edit-2509-Pruning.jpg", + "desc": "This open-source project is based on Qwen-Image-Edit and has attempted model pruning, removing 20 layers while retaining the weights of 40 layers, resulting in a model size of 13.6B parameters.", + "skip": true, + "date": "2025 Ocotober" + }, + "lodestones Chroma1 HD": { "path": "lodestones/Chroma1-HD", "preview": "lodestones--Chroma-HD.jpg", "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. This is the high-res fine-tune of the Chroma1-Base at a 1024x1024 resolution.", "skip": true, - "extras": "" + "extras": "", + "size": 26.84, + "date": "2025 July" }, "lodestones Chroma1 Base": { "path": "lodestones/Chroma1-Base", "preview": "lodestones--Chroma-Base.jpg", "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. This is the core 512x512 model. It's a solid, all-around foundation for pretty much any creative project.", "skip": true, - "extras": "" + "extras": "", + "size": 26.84, + "date": "2025 July" }, "lodestones Chroma1 Flash": { "path": "lodestones/Chroma1-Flash", "preview": "lodestones--Chroma-flash.jpg", "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. A fine-tuned version of the Chroma1-Base made to find the best way to make these flow matching models faster.", "skip": true, - "extras": "" + "extras": "", + "size": 26.84, + "date": "2025 July" }, "lodestones Chroma1 v50 Preview Annealed": { "path": "vladmandic/chroma-unlocked-v50-annealed", "preview": "lodestones--Chroma-annealed.jpg", "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. Re-tweaked variant with extra noise added.", "skip": true, - "extras": "" + "extras": "", + "size": 26.84, + "date": "2025 July" }, "lodestones Chroma1 v48 Preview": { "path": "vladmandic/chroma-unlocked-v48", "preview": "lodestones--Chroma.jpg", "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. Last raw version of Chroma before final finetuning.", "skip": true, - "extras": "" + "extras": "", + "size": 26.84, + "date": "2025 July" }, "lodestones Chroma1 v48 Preview Calibrated": { "path": "vladmandic/chroma-unlocked-v48-detail-calibrated", "preview": "lodestones--Chroma-detail.jpg", "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. Last raw version of Chroma before final finetuning but with some detail calibration.", "skip": true, - "extras": "" - }, - - "Qwen-Image": { - "path": "Qwen/Qwen-Image", - "preview": "Qwen--Qwen-Image.jpg", - "desc": " Qwen-Image, an image generation foundation model in the Qwen series that achieves significant advances in complex text rendering and precise image editing.", - "skip": true, - "extras": "" - }, - "Qwen-Image-Edit": { - "path": "Qwen/Qwen-Image-Edit", - "preview": "Qwen--Qwen-Image-Edit.jpg", - "desc": "Qwen-Image-Edit, the image editing version of Qwen-Image. Built upon our 20B Qwen-Image model, Qwen-Image-Edit successfully extends Qwen-Image’s unique text rendering capabilities to image editing tasks, enabling precise text editing.", - "skip": true, - "extras": "" - }, - "Qwen-Image-Lightning": { - "path": "vladmandic/Qwen-Lightning", - "preview": "vladmandic--Qwen-Lightning.jpg", - "desc": "Qwen-Lightning is step-distilled from Qwen-Image to allow for generation in 8 steps.", - "skip": true, - "extras": "steps: 8" - }, - "Qwen-Image-Distill": { - "path": "SahilCarterr/Qwen-Image-Distill-Full", - "preview": "SahilCarterr--Qwen-Image-Distill-Full.jpg", - "desc": "Qwen-Image-Distill is a distilled and accelerated version of Qwen-Image by DiffSynth-Studio.", - "skip": true, - "extras": "steps: 15" - }, - "Qwen-Image-Lightning-Edit": { - "path": "vladmandic/Qwen-Lightning-Edit", - "preview": "vladmandic--Qwen-Lightning-Edit.jpg", - "desc": " Qwen-Lightning-Edit is step-distilled from Qwen-Image-Edit to allow for generation in 8 steps.", - "skip": true, - "extras": "steps: 8" + "extras": "", + "size": 26.84, + "date": "2025 July" }, "Ostris Flex.2 Preview": { @@ -223,14 +319,18 @@ "preview": "ostris--Flex.2-preview.jpg", "desc": "Open Source 8B parameter Text to Image Diffusion Model with universal control and inpainting support built in. Early access preview release. The next version of Flex.1-alpha", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 25.65, + "date": "2025 April" }, "Ostris Flex.1 Alpha": { "path": "ostris/Flex.1-alpha", "preview": "ostris--Flex.1-alpha.jpg", "desc": "Flex.1 alpha is a pre-trained base 8 billion parameter rectified flow transformer capable of generating images from text descriptions. It has a similar architecture to FLUX.1-dev, but with fewer double transformer blocks (8 vs 19)", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 25.65, + "date": "2025 January" }, "Wan-AI Wan2.1 1.3B": { @@ -238,14 +338,18 @@ "preview": "Wan-AI--Wan2.1-T2V-1.3B-Diffusers.jpg", "desc": "Wan is an advanced and powerful visual generation model developed by Tongyi Lab of Alibaba Group. It can generate videos based on text, images, and other control signals. The Wan2.1 series models are now fully open-source.", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 27.72, + "date": "2025 February" }, "Wan-AI Wan2.1 14B": { "path": "Wan-AI/Wan2.1-T2V-14B-Diffusers", "preview": "Wan-AI--Wan2.1-T2V-14B-Diffusers.jpg", "desc": "Wan is an advanced and powerful visual generation model developed by Tongyi Lab of Alibaba Group. It can generate videos based on text, images, and other control signals. The Wan2.1 series models are now fully open-source.", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 78.52, + "date": "2025 February" }, "Wan-AI Wan2.2 5B": { "path": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", @@ -268,27 +372,40 @@ "skip": true, "extras": "sampler: Default" }, + "Wan-AI Wan2.2 14B VACE": { + "path": "linoyts/Wan2.2-VACE-Fun-14B-diffusers", + "preview": "linoyts--Wan2.2-VACE-Fun-14B-diffusers.jpg", + "desc": "Wan2.2, offering more powerful capabilities, better performance, and superior visual quality. With Wan2.2, we have focused on incorporating the following technical innovations: MoE Architecture, Data Scalling, Cinematic Aesthetics, Efficient High-Definition Hybrid", + "skip": true, + "extras": "sampler: Default" + }, "Freepik F-Lite": { "path": "Freepik/F-Lite", "preview": "Freepik--F-Lite.jpg", "desc": "F Lite is a 10B parameter diffusion model created by Freepik and Fal, trained exclusively on copyright-safe and SFW content. The model was trained on Freepik's internal dataset comprising approximately 80 million copyright-safe images, making it the first publicly available model of this scale trained exclusively on legally compliant and SFW content.", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 19.81, + "date": "2025 May" }, "Freepik F-Lite Texture": { "path": "Freepik/F-Lite-Texture", "preview": "Freepik--F-Lite-Texture.jpg", "desc": "F Lite is a 10B parameter diffusion model created by Freepik and Fal, trained exclusively on copyright-safe and SFW content. The model was trained on Freepik's internal dataset comprising approximately 80 million copyright-safe images, making it the first publicly available model of this scale trained exclusively on legally compliant and SFW content.", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 19.81, + "date": "2025 May" }, "Freepik F-Lite 7B": { "path": "Freepik/F-Lite-7B", "preview": "Freepik--F-Lite-7B.jpg", "desc": "F Lite is a 10B parameter diffusion model created by Freepik and Fal, trained exclusively on copyright-safe and SFW content. The model was trained on Freepik's internal dataset comprising approximately 80 million copyright-safe images, making it the first publicly available model of this scale trained exclusively on legally compliant and SFW content.", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 13.89, + "date": "2025 May" }, "SDXS DreamShaper 512": { @@ -309,13 +426,17 @@ "path": "Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers", "desc": "Sana is an efficient model with scaling of training-time and inference time techniques. SANA-1.5 delivers: efficient model growth from 1.6B Sana-1.0 model to 4.8B, achieving similar or better performance than training from scratch and saving 60% training cost; efficient model depth pruning, slimming any model size as you want; powerful VLM selection based inference scaling, smaller model+inference scaling > larger model.", "preview": "Efficient-Large-Model--SANA1.5_1.6B_1024px_diffusers.jpg", - "skip": true + "skip": true, + "size": 9.49, + "date": "2025 March" }, "NVLabs Sana 1.5 4.8B 1k": { "path": "Efficient-Large-Model/SANA1.5_4.8B_1024px_diffusers", "desc": "Sana is an efficient model with scaling of training-time and inference time techniques. SANA-1.5 delivers: efficient model growth from 1.6B Sana-1.0 model to 4.8B, achieving similar or better performance than training from scratch and saving 60% training cost; efficient model depth pruning, slimming any model size as you want; powerful VLM selection based inference scaling, smaller model+inference scaling > larger model.", "preview": "Efficient-Large-Model--SANA1.5_4.8B_1024px_diffusers.jpg", - "skip": true + "skip": true, + "size": 15.58, + "date": "2025 March" }, "NVLabs Sana 1.5 1.6B 1k Sprint": { "path": "Efficient-Large-Model/Sana_Sprint_1.6B_1024px_diffusers", @@ -327,63 +448,93 @@ "path": "Efficient-Large-Model/Sana_1600M_4Kpx_BF16_diffusers", "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", "preview": "Efficient-Large-Model--Sana_1600M_4Kpx_BF16_diffusers.jpg", - "skip": true + "skip": true, + "size": 12.63, + "date": "2024 November" }, "NVLabs Sana 1.0 1.6B 2k": { "path": "Efficient-Large-Model/Sana_1600M_2Kpx_BF16_diffusers", "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", "preview": "Efficient-Large-Model--Sana_1600M_2Kpx_BF16_diffusers.jpg", - "skip": true + "skip": true, + "size": 12.63, + "date": "2024 November" }, "NVLabs Sana 1.0 1.6B 1k": { "path": "Efficient-Large-Model/Sana_1600M_1024px_diffusers", "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", "preview": "Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg", - "skip": true + "skip": true, + "size": 12.63, + "date": "2024 November" }, "NVLabs Sana 1.0 0.6B 0.5k": { "path": "Efficient-Large-Model/Sana_600M_512px_diffusers", "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", "preview": "Efficient-Large-Model--Sana_600M_512px_diffusers.jpg", - "skip": true + "skip": true, + "size": 7.51, + "date": "2024 November" }, "nVidia Cosmos-Predict2 T2I 2B": { "path": "nvidia/Cosmos-Predict2-2B-Text2Image", "desc": "Cosmos-Predict2: A family of highly performant pre-trained world foundation models purpose-built for generating physics-aware images, videos and world states for physical AI development.", "preview": "nvidia--Cosmos-Predict2-2B-Text2Image.jpg", - "skip": true + "skip": true, + "size": 13.32, + "date": "2025 June" }, "nVidia Cosmos-Predict2 T2I 14B": { "path": "nvidia/Cosmos-Predict2-14B-Text2Image", "desc": "Cosmos-Predict2: A family of highly performant pre-trained world foundation models purpose-built for generating physics-aware images, videos and world states for physical AI development.", "preview": "nvidia--Cosmos-Predict2-14B-Text2Image.jpg", - "skip": true + "skip": true, + "size": 37.36, + "date": "2025 June" + }, + + "X-Omni SFT": { + "path": "X-Omni/X-Omni-SFT", + "desc": "X-Omni: Reinforcement learning makes discrete autoregressive image generative models great again", + "preview": "X-Omni--X-Omni-SFT.jpg", + "skip": true, + "size": 0, + "date": "2024 September", + "experimental": true }, "VectorSpaceLab OmniGen v1": { "path": "Shitao/OmniGen-v1-diffusers", "desc": "OmniGen is a unified image generation model that can generate a wide range of images from multi-modal prompts. It is designed to be simple, flexible and easy to use.", "preview": "Shitao--OmniGen-v1.jpg", - "skip": true + "skip": true, + "size": 15.47, + "date": "2024 October" }, "VectorSpaceLab OmniGen v2": { "path": "OmniGen2/OmniGen2", "desc": "OmniGen2 is a powerful and efficient unified multimodal model. Unlike OmniGen v1, OmniGen2 features two distinct decoding pathways for text and image modalities, utilizing unshared parameters and a decoupled image tokenizer.", "preview": "OmniGen2--OmniGen2.jpg", - "skip": true + "skip": true, + "size": 30.5, + "date": "2025 June" }, "AuraFlow 0.3": { "path": "fal/AuraFlow-v0.3", "desc": "AuraFlow v0.3 is the fully open-sourced flow-based text-to-image generation model. The model was trained with more compute compared to the previous version, AuraFlow-v0.2. Compared to AuraFlow-v0.2, the model is fine-tuned on more aesthetic datasets and now supports various aspect ratio, (now width and height up to 1536 pixels).", "preview": "fal--AuraFlow-v0.3.jpg", - "skip": true + "skip": true, + "size": 31.9, + "date": "2024 August" }, "AuraFlow 0.2": { "path": "fal/AuraFlow-v0.2", "desc": "AuraFlow v0.2 is the fully open-sourced largest flow-based text-to-image generation model. The model was trained with more compute compared to the previous version, AuraFlow-v0.1", "preview": "fal--AuraFlow-v0.2.jpg", - "skip": true + "skip": true, + "size": 31.9, + "date": "2024 July" }, "Segmind Vega": { @@ -392,7 +543,9 @@ "desc": "The Segmind-Vega Model is a distilled version of the Stable Diffusion XL (SDXL), offering a remarkable 70% reduction in size and an impressive 100% speedup while retaining high-quality text-to-image generation capabilities. Trained on diverse datasets, including Grit and Midjourney scrape data, it excels at creating a wide range of visual content based on textual prompts. Employing a knowledge distillation strategy, Segmind-Vega leverages the teachings of several expert models, including SDXL, ZavyChromaXL, and JuggernautXL, to combine their strengths and produce compelling visual outputs.", "variant": "fp16", "skip": true, - "extras": "sampler: Default, cfg_scale: 9.0" + "extras": "sampler: Default, cfg_scale: 9.0", + "size": 6.43, + "date": "2023 November" }, "Segmind SSD-1B": { "path": "huggingface/segmind/SSD-1B", @@ -400,13 +553,17 @@ "desc": "The Segmind Stable Diffusion Model (SSD-1B) offers a compact, efficient, and distilled version of the SDXL model. At 50% smaller and 60% faster than Stable Diffusion XL (SDXL), it provides quick and seamless performance without sacrificing image quality.", "variant": "fp16", "skip": true, - "extras": "sampler: Default, cfg_scale: 9.0" + "extras": "sampler: Default, cfg_scale: 9.0", + "size": 8.72, + "date": "2023 October" }, "Segmind Tiny": { "path": "segmind/tiny-sd", "preview": "segmind--tiny-sd.jpg", "desc": "Segmind's Tiny-SD offers a compact, efficient, and distilled version of Realistic Vision 4.0 and is up to 80% faster than SD1.5", - "extras": "width: 512, height: 512, sampler: Default, cfg_scale: 9.0" + "extras": "width: 512, height: 512, sampler: Default, cfg_scale: 9.0", + "size": 1.03, + "date": "2023 July" }, "Segmind SegMoE SD 4x2": { "path": "segmind/SegMoE-SD-4x2-v0", @@ -430,7 +587,9 @@ "path": "PixArt-alpha/PixArt-XL-2-1024-MS", "desc": "PixArt-α is a Transformer-based T2I diffusion model whose image generation quality is competitive with state-of-the-art image generators (e.g., Imagen, SDXL, and even Midjourney), and the training speed markedly surpasses existing large-scale T2I models. Extensive experiments demonstrate that PIXART-α excels in image quality, artistry, and semantic control. It can directly generate 1024px images from text prompts within a single sampling process.", "preview": "PixArt-alpha--PixArt-XL-2-1024-MS.jpg", - "extras": "sampler: Default, cfg_scale: 2.0" + "extras": "sampler: Default, cfg_scale: 2.0", + "size": 21.3, + "date": "2023 November" }, "Pixart-Σ Small": { "path": "huggingface/PixArt-alpha/PixArt-Sigma-XL-2-512-MS", @@ -451,14 +610,18 @@ "desc": "PixArt-Σ, a Diffusion Transformer model (DiT) capable of directly generating images at 4K resolution. PixArt-Σ represents a significant advancement over its predecessor, PixArt-α, offering images of markedly higher fidelity and improved alignment with text prompts.", "preview": "PixArt-alpha--PixArt-Sigma-XL-2-2K-MS.jpg", "skip": true, - "extras": "sampler: Default, cfg_scale: 2.0" + "extras": "sampler: Default, cfg_scale: 2.0", + "size": 21.3, + "date": "2024 April" }, "Tencent HunyuanDiT 1.2": { "path": "Tencent-Hunyuan/HunyuanDiT-v1.2-Diffusers", "desc": "Hunyuan-DiT : A Powerful Multi-Resolution Diffusion Transformer with Fine-Grained Chinese Understanding.", "preview": "Tencent-Hunyuan--HunyuanDiT-v1.2-Diffusers.jpg", - "extras": "sampler: Default, cfg_scale: 2.0" + "extras": "sampler: Default, cfg_scale: 2.0", + "size": 14.09, + "date": "2024 May" }, "Tencent HunyuanDiT 1.2 Distilled": { "path": "Tencent-Hunyuan/HunyuanDiT-v1.2-Diffusers-Distilled", @@ -484,14 +647,18 @@ "desc": "The Lumina-Next-SFT is a Next-DiT model containing 2B parameters and utilizes Gemma-2B as the text encoder, enhanced through high-quality supervised fine-tuning (SFT).", "preview": "Alpha-VLLM--Lumina-Next-SFT-diffusers.jpg", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 8.67, + "date": "2024 June" }, "AlphaVLLM Lumina 2": { "path": "Alpha-VLLM/Lumina-Image-2.0", "desc": "A Unified and Efficient Image Generative Model. Lumina-Image-2.0 is a 2 billion parameter flow-based diffusion transformer capable of generating images from text descriptions.", "preview": "Alpha-VLLM--Lumina-Image-2.0.jpg", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 20.75, + "date": "2025 January" }, "HiDream-I1 Fast": { @@ -499,21 +666,27 @@ "desc": "HiDream-I1 is a new open-source image generative foundation model with 17B parameters that achieves state-of-the-art image generation quality within seconds.", "preview": "HiDream-ai--HiDream-I1-Fast.jpg", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 58.4, + "date": "2025 April" }, "HiDream-I1 Dev": { "path": "HiDream-ai/HiDream-I1-Dev", "desc": "HiDream-I1 is a new open-source image generative foundation model with 17B parameters that achieves state-of-the-art image generation quality within seconds.", "preview": "HiDream-ai--HiDream-I1-Dev.jpg", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 58.4, + "date": "2025 April" }, "HiDream-I1 Full": { "path": "HiDream-ai/HiDream-I1-Full", "desc": "HiDream-I1 is a new open-source image generative foundation model with 17B parameters that achieves state-of-the-art image generation quality within seconds.", "preview": "HiDream-ai--HiDream-I1-Full.jpg", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 58.4, + "date": "2025 April" }, "HiDream-E1 Full": { "path": "HiDream-ai/HiDream-E1-Full", @@ -522,40 +695,57 @@ "skip": true, "extras": "sampler: Default" }, + "HiDream-E1.1": { + "path": "HiDream-ai/HiDream-E1-1", + "desc": "HiDream-E1 is an image editing model built on HiDream-I1.", + "preview": "HiDream-ai--HiDream-E1-Full.jpg", + "skip": true, + "extras": "sampler: Default" + }, "Kwai Kolors": { "path": "Kwai-Kolors/Kolors-diffusers", "desc": "Kolors is a large-scale text-to-image generation model based on latent diffusion, developed by the Kuaishou Kolors team. Trained on billions of text-image pairs, Kolors exhibits significant advantages over both open-source and proprietary models in visual quality, complex semantic accuracy, and text rendering for both Chinese and English characters. Furthermore, Kolors supports both Chinese and English inputs", "preview": "Kwai-Kolors--Kolors-diffusers.jpg", "skip": true, - "extras": "width: 1024, height: 1024" + "extras": "width: 1024, height: 1024", + "size": 17.40, + "date": "2024 July" }, "Kandinsky 2.1": { "path": "kandinsky-community/kandinsky-2-1", "desc": "Kandinsky 2.1 is a text-conditional diffusion model based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder. Kandinsky 2.1 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas. It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.", "preview": "kandinsky-community--kandinsky-2-1.jpg", - "extras": "width: 768, height: 768, sampler: Default" + "extras": "width: 768, height: 768, sampler: Default", + "size": 5.15, + "date": "2023 April" }, "Kandinsky 2.2": { "path": "kandinsky-community/kandinsky-2-2-decoder", "desc": "Kandinsky 2.2 is a text-conditional diffusion model (+0.1!) based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder. Kandinsky 2.2 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas. It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.", "preview": "kandinsky-community--kandinsky-2-2-decoder.jpg", - "extras": "width: 768, height: 768, sampler: Default" + "extras": "width: 768, height: 768, sampler: Default", + "size": 5.15, + "date": "2023 July" }, "Kandinsky 3": { "path": "kandinsky-community/kandinsky-3", "desc": "Kandinsky 3.0 is an open-source text-to-image diffusion model built upon the Kandinsky2-x model family. In comparison to its predecessors, Kandinsky 3.0 incorporates more data and specifically related to Russian culture, which allows to generate pictures related to Russin culture. Furthermore, enhancements have been made to the text understanding and visual quality of the model, achieved by increasing the size of the text encoder and Diffusion U-Net models, respectively.", "preview": "kandinsky-community--kandinsky-3.jpg", "variant": "fp16", - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 27.72, + "date": "2023 November" }, "Playground v1": { "path": "playgroundai/playground-v1", "desc": "Playground v1 is a latent diffusion model that improves the overall HDR quality to get more stunning images.", "preview": "playgroundai--playground-v1.jpg", - "extras": "width: 512, height: 512, sampler: Default" + "extras": "width: 512, height: 512, sampler: Default", + "size": 4.95, + "date": "2023 December" }, "Playground v2 Small": { "path": "playgroundai/playground-v2-256px-base", @@ -578,22 +768,28 @@ "Playground v2.5": { "path": "playgroundai/playground-v2.5-1024px-aesthetic", "desc": "Playground v2.5 is a diffusion-based text-to-image generative model, and a successor to Playground v2. Playground v2.5 is the state-of-the-art open-source model in aesthetic quality.", - "preview": "playgroundai--playground-v2_5-1024px-aesthetic.jpg", + "preview": "playgroundai--playground-v2.5-1024px-aesthetic.jpg", "variant": "fp16", - "extras": "sampler: DPM++ 2M EDM" + "extras": "sampler: DPM++ 2M EDM", + "size": 13.35, + "date": "2023 December" }, "CogView 4": { "path": "zai-org/CogView4-6B", "desc": "An innovative cascaded framework that enhances the performance of text-to-image diffusion. CogView is the first model implementing relay diffusion in the realm of text-to-image generation, executing the task by first creating low-resolution images and subsequently applying relay-based super-resolution.", "preview": "THUDM--CogView4-6B.jpg", - "skip": true + "skip": true, + "size": 30.39, + "date": "2025 March" }, "CogView 3 Plus": { "path": "zai-org/CogView3-Plus-3B", "desc": "An innovative cascaded framework that enhances the performance of text-to-image diffusion. CogView is the first model implementing relay diffusion in the realm of text-to-image generation, executing the task by first creating low-resolution images and subsequently applying relay-based super-resolution.", "preview": "THUDM--CogView3-Plus-3B.jpg", - "skip": true + "skip": true, + "size": 24.96, + "date": "2024 October" }, "ShuttleAI Shuttle 3.0 Diffusion": { @@ -619,14 +815,18 @@ "path": "briaai/BRIA-3.2", "desc": "Bria 3.2 is the next-generation commercial-ready text-to-image model. With just 4 billion parameters, it provides exceptional aesthetics and text rendering, evaluated to provide on par results to leading open-source models, and outperforming other licensed models.", "preview": "briaai--BRIA-3.2.jpg", - "skip": true + "skip": true, + "size": 18.66, + "date": "2025 June" }, "Meissonic": { "path": "MeissonFlow/Meissonic", "desc": "Meissonic is a non-autoregressive mask image modeling text-to-image synthesis model that can generate high-resolution images. It is designed to run on consumer graphics cards.", "preview": "MeissonFlow--Meissonic.jpg", - "skip": true + "skip": true, + "size": 3.64, + "date": "2024 October" }, "aMUSEd 256": { @@ -647,7 +847,9 @@ "path": "warp-ai/wuerstchen", "desc": "Würstchen is a diffusion model whose text-conditional model works in a highly compressed latent space of images. Why is this important? Compressing data can reduce computational costs for both training and inference by magnitudes. Training on 1024x1024 images, is way more expensive than training at 32x32. Usually, other works make use of a relatively small compression, in the range of 4x - 8x spatial compression. Würstchen takes this to an extreme. Through its novel design, we achieve a 42x spatial compression. Würstchen employs a two-stage compression, what we call Stage A and Stage B. Stage A is a VQGAN, and Stage B is a Diffusion Autoencoder (more details can be found in the paper). A third model, Stage C, is learned in that highly compressed latent space. This training requires fractions of the compute used for current top-performing models, allowing also cheaper and faster inference.", "preview": "warp-ai--wuerstchen.jpg", - "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 0.0" + "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 0.0", + "size": 12.16, + "date": "2023 August" }, "KOALA 700M": { @@ -656,7 +858,9 @@ "skip": true, "desc": "Fast text-to-image model, called KOALA, by compressing SDXL's U-Net and distilling knowledge from SDXL into our model. KOALA-700M can generate a 1024x1024 image in less than 1.5 seconds on an NVIDIA 4090 GPU, which is more than 2x faster than SDXL.", "preview": "etri-vilab--koala-700m-llava-cap.jpg", - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 6.58, + "date": "2024 January" }, "HDM-XUT 340M Anime": { @@ -671,13 +875,17 @@ "path": "thu-ml/unidiffuser-v1", "desc": "UniDiffuser is a unified diffusion framework to fit all distributions relevant to a set of multi-modal data in one transformer. UniDiffuser is able to perform image, text, text-to-image, image-to-text, and image-text pair generation by setting proper timesteps without additional overhead.\nSpecifically, UniDiffuser employs a variation of transformer, called U-ViT, which parameterizes the joint noise prediction network. Other components perform as encoders and decoders of different modalities, including a pretrained image autoencoder from Stable Diffusion, a pretrained image ViT-B/32 CLIP encoder, a pretrained text ViT-L CLIP encoder, and a GPT-2 text decoder finetuned by ourselves.", "preview": "thu-ml--unidiffuser-v1.jpg", - "extras": "width: 512, height: 512, sampler: Default" + "extras": "width: 512, height: 512, sampler: Default", + "size": 5.37, + "date": "2023 May" }, "SalesForce BLIP-Diffusion": { "path": "salesforce/blipdiffusion", "desc": "BLIP-Diffusion, a new subject-driven image generation model that supports multimodal control which consumes inputs of subject images and text prompts. Unlike other subject-driven generation models, BLIP-Diffusion introduces a new multimodal encoder which is pre-trained to provide subject representation.", - "preview": "salesforce--blipdiffusion.jpg" + "preview": "salesforce--blipdiffusion.jpg", + "size": 7.23, + "date": "2023 July" }, "InstaFlow 0.9B": { @@ -690,13 +898,17 @@ "path": "DeepFloyd/IF-I-M-v1.0", "desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.", "preview": "DeepFloyd--IF-I-M-v1.0.jpg", - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 12.79, + "date": "2023 April" }, "DeepFloyd IF Large": { "path": "DeepFloyd/IF-I-L-v1.0", "desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.", "preview": "DeepFloyd--IF-I-L-v1.0.jpg", - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 15.48, + "date": "2023 April" } } diff --git a/html/swagger.css b/html/swagger.css index 77a578923..ea5bc261b 100644 --- a/html/swagger.css +++ b/html/swagger.css @@ -1,868 +1,10499 @@ -.opblock { - border-width: 0 !important; -} -.opblock-summary-operation-id { - display: none !important; -} -.swagger-ui .models .json-schema-2020-12:not(.json-schema-2020-12--embedded)>.json-schema-2020-12-head .json-schema-2020-12__title:first-of-type { - font-size: 14px; - color: white; +html { + background-color: #aaa; } -.swagger-ui .json-schema-2020-12-keyword__name--primary { - color: aqua; +.swagger-ui { + color: #3b4151; + font-family: system-ui, sans-serif; +} + +.swagger-ui body { + margin: 0; +} + +.swagger-ui article,.swagger-ui aside,.swagger-ui footer,.swagger-ui header,.swagger-ui nav,.swagger-ui section { + display: block +} + +.swagger-ui h1 { + font-size: 2em; + margin: .67em 0 +} + +.swagger-ui figcaption,.swagger-ui figure,.swagger-ui main { + display: block +} + +.swagger-ui figure { + margin: 1em 40px +} + +.swagger-ui hr { + box-sizing: content-box; + height: 0; + overflow: visible +} + +.swagger-ui pre { + font-family: monospace,monospace; + font-size: 1em +} + +.swagger-ui a { + background-color: transparent; + -webkit-text-decoration-skip: objects +} + +.swagger-ui abbr[title] { + text-decoration: underline; + -webkit-text-decoration: underline dotted; + text-decoration: underline dotted +} + +.swagger-ui b,.swagger-ui strong { + font-weight: inherit; + font-weight: bolder +} + +.swagger-ui code,.swagger-ui kbd,.swagger-ui samp { + font-family: monospace,monospace; + font-size: 1em +} + +.swagger-ui dfn { + font-style: italic +} + +.swagger-ui mark { + background-color: #ff0; + color: #000 +} + +.swagger-ui small { + font-size: 80% +} + +.swagger-ui sub,.swagger-ui sup { + font-size: 75%; + line-height: 0; + position: relative; + vertical-align: baseline +} + +.swagger-ui sub { + bottom: -.25em +} + +.swagger-ui sup { + top: -.5em +} + +.swagger-ui audio,.swagger-ui video { + display: inline-block +} + +.swagger-ui audio:not([controls]) { + display: none; + height: 0 +} + +.swagger-ui svg:not(:root) { + overflow: hidden +} + +.swagger-ui button,.swagger-ui input,.swagger-ui optgroup,.swagger-ui select,.swagger-ui textarea { + font-family: sans-serif; + font-size: 100%; + line-height: 1.15; + margin: 0 +} + +.swagger-ui button,.swagger-ui input { + overflow: visible +} + +.swagger-ui button,.swagger-ui select { + text-transform: none +} + +.swagger-ui [type=reset],.swagger-ui [type=submit],.swagger-ui button,.swagger-ui html [type=button] { + -webkit-appearance: button +} + +.swagger-ui [type=button]::-moz-focus-inner,.swagger-ui [type=reset]::-moz-focus-inner,.swagger-ui [type=submit]::-moz-focus-inner,.swagger-ui button::-moz-focus-inner { + padding: 0 +} + +.swagger-ui [type=button]:-moz-focusring,.swagger-ui [type=reset]:-moz-focusring,.swagger-ui [type=submit]:-moz-focusring,.swagger-ui button:-moz-focusring { + outline: 1px dotted ButtonText +} + +.swagger-ui fieldset { + padding: .35em .75em .625em +} + +.swagger-ui legend { + box-sizing: border-box; + color: inherit; + display: table; + max-width: 100%; + padding: 0; + white-space: normal +} + +.swagger-ui progress { + display: inline-block; + vertical-align: baseline +} + +.swagger-ui textarea { + overflow: auto +} + +.swagger-ui [type=checkbox],.swagger-ui [type=radio] { + box-sizing: border-box; + padding: 0 +} + +.swagger-ui [type=number]::-webkit-inner-spin-button,.swagger-ui [type=number]::-webkit-outer-spin-button { + height: auto +} + +.swagger-ui [type=search] { + -webkit-appearance: textfield; + outline-offset: -2px +} + +.swagger-ui [type=search]::-webkit-search-cancel-button,.swagger-ui [type=search]::-webkit-search-decoration { + -webkit-appearance: none +} + +.swagger-ui ::-webkit-file-upload-button { + -webkit-appearance: button; + font: inherit +} + +.swagger-ui details,.swagger-ui menu { + display: block +} + +.swagger-ui summary { + display: list-item +} + +.swagger-ui canvas { + display: inline-block +} + +.swagger-ui [hidden],.swagger-ui template { + display: none +} + +.swagger-ui .debug * { + outline: 1px solid gold +} + +.swagger-ui .debug-white * { + outline: 1px solid #fff +} + +.swagger-ui .debug-black * { + outline: 1px solid #000 +} + +.swagger-ui .debug-grid { + background: transparent url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMTExIDc5LjE1ODMyNSwgMjAxNS8wOS8xMC0wMToxMDoyMCAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6MTRDOTY4N0U2N0VFMTFFNjg2MzZDQjkwNkQ4MjgwMEIiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6MTRDOTY4N0Q2N0VFMTFFNjg2MzZDQjkwNkQ4MjgwMEIiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3NjcyQkQ3NjY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3NjcyQkQ3NzY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PsBS+GMAAAAjSURBVHjaYvz//z8DLsD4gcGXiYEAGBIKGBne//fFpwAgwAB98AaF2pjlUQAAAABJRU5ErkJggg==) repeat 0 0 +} + +.swagger-ui .debug-grid-16 { + background: transparent url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMTExIDc5LjE1ODMyNSwgMjAxNS8wOS8xMC0wMToxMDoyMCAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6ODYyRjhERDU2N0YyMTFFNjg2MzZDQjkwNkQ4MjgwMEIiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6ODYyRjhERDQ2N0YyMTFFNjg2MzZDQjkwNkQ4MjgwMEIiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3NjcyQkQ3QTY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3NjcyQkQ3QjY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PvCS01IAAABMSURBVHjaYmR4/5+BFPBfAMFm/MBgx8RAGWCn1AAmSg34Q6kBDKMGMDCwICeMIemF/5QawEipAWwUhwEjMDvbAWlWkvVBwu8vQIABAEwBCph8U6c0AAAAAElFTkSuQmCC) repeat 0 0 +} + +.swagger-ui .debug-grid-8-solid { + background: #fff url(data:image/jpeg;base64,/9j/4QAYRXhpZgAASUkqAAgAAAAAAAAAAAAAAP/sABFEdWNreQABAAQAAAAAAAD/4QMxaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wLwA8P3hwYWNrZXQgYmVnaW49Iu+7vyIgaWQ9Ilc1TTBNcENlaGlIenJlU3pOVGN6a2M5ZCI/PiA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJBZG9iZSBYTVAgQ29yZSA1LjYtYzExMSA3OS4xNTgzMjUsIDIwMTUvMDkvMTAtMDE6MTA6MjAgICAgICAgICI+IDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+IDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiIHhtbG5zOnhtcD0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wLyIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bXA6Q3JlYXRvclRvb2w9IkFkb2JlIFBob3Rvc2hvcCBDQyAyMDE1IChNYWNpbnRvc2gpIiB4bXBNTTpJbnN0YW5jZUlEPSJ4bXAuaWlkOkIxMjI0OTczNjdCMzExRTZCMkJDRTI0MDgxMDAyMTcxIiB4bXBNTTpEb2N1bWVudElEPSJ4bXAuZGlkOkIxMjI0OTc0NjdCMzExRTZCMkJDRTI0MDgxMDAyMTcxIj4gPHhtcE1NOkRlcml2ZWRGcm9tIHN0UmVmOmluc3RhbmNlSUQ9InhtcC5paWQ6QjEyMjQ5NzE2N0IzMTFFNkIyQkNFMjQwODEwMDIxNzEiIHN0UmVmOmRvY3VtZW50SUQ9InhtcC5kaWQ6QjEyMjQ5NzI2N0IzMTFFNkIyQkNFMjQwODEwMDIxNzEiLz4gPC9yZGY6RGVzY3JpcHRpb24+IDwvcmRmOlJERj4gPC94OnhtcG1ldGE+IDw/eHBhY2tldCBlbmQ9InIiPz7/7gAOQWRvYmUAZMAAAAAB/9sAhAAbGhopHSlBJiZBQi8vL0JHPz4+P0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHAR0pKTQmND8oKD9HPzU/R0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0f/wAARCAAIAAgDASIAAhEBAxEB/8QAWQABAQAAAAAAAAAAAAAAAAAAAAYBAQEAAAAAAAAAAAAAAAAAAAIEEAEBAAMBAAAAAAAAAAAAAAABADECA0ERAAEDBQAAAAAAAAAAAAAAAAARITFBUWESIv/aAAwDAQACEQMRAD8AoOnTV1QTD7JJshP3vSM3P//Z) repeat 0 0 +} + +.swagger-ui .debug-grid-16-solid { + background: #fff url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMTExIDc5LjE1ODMyNSwgMjAxNS8wOS8xMC0wMToxMDoyMCAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6NzY3MkJEN0U2N0M1MTFFNkIyQkNFMjQwODEwMDIxNzEiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6NzY3MkJEN0Y2N0M1MTFFNkIyQkNFMjQwODEwMDIxNzEiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3NjcyQkQ3QzY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3NjcyQkQ3RDY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/Pve6J3kAAAAzSURBVHjaYvz//z8D0UDsMwMjSRoYP5Gq4SPNbRjVMEQ1fCRDg+in/6+J1AJUxsgAEGAA31BAJMS0GYEAAAAASUVORK5CYII=) repeat 0 0 +} + +.swagger-ui .border-box,.swagger-ui a,.swagger-ui article,.swagger-ui body,.swagger-ui code,.swagger-ui dd,.swagger-ui div,.swagger-ui dl,.swagger-ui dt,.swagger-ui fieldset,.swagger-ui footer,.swagger-ui form,.swagger-ui h1,.swagger-ui h2,.swagger-ui h3,.swagger-ui h4,.swagger-ui h5,.swagger-ui h6,.swagger-ui header,.swagger-ui html,.swagger-ui input[type=email],.swagger-ui input[type=number],.swagger-ui input[type=password],.swagger-ui input[type=tel],.swagger-ui input[type=text],.swagger-ui input[type=url],.swagger-ui legend,.swagger-ui li,.swagger-ui main,.swagger-ui ol,.swagger-ui p,.swagger-ui pre,.swagger-ui section,.swagger-ui table,.swagger-ui td,.swagger-ui textarea,.swagger-ui th,.swagger-ui tr,.swagger-ui ul { + box-sizing: border-box +} + +.swagger-ui .aspect-ratio { + height: 0; + position: relative +} + +.swagger-ui .aspect-ratio--16x9 { + padding-bottom: 56.25% +} + +.swagger-ui .aspect-ratio--9x16 { + padding-bottom: 177.77% +} + +.swagger-ui .aspect-ratio--4x3 { + padding-bottom: 75% +} + +.swagger-ui .aspect-ratio--3x4 { + padding-bottom: 133.33% +} + +.swagger-ui .aspect-ratio--6x4 { + padding-bottom: 66.6% +} + +.swagger-ui .aspect-ratio--4x6 { + padding-bottom: 150% +} + +.swagger-ui .aspect-ratio--8x5 { + padding-bottom: 62.5% +} + +.swagger-ui .aspect-ratio--5x8 { + padding-bottom: 160% +} + +.swagger-ui .aspect-ratio--7x5 { + padding-bottom: 71.42% +} + +.swagger-ui .aspect-ratio--5x7 { + padding-bottom: 140% +} + +.swagger-ui .aspect-ratio--1x1 { + padding-bottom: 100% +} + +.swagger-ui .aspect-ratio--object { + bottom: 0; + height: 100%; + left: 0; + position: absolute; + right: 0; + top: 0; + width: 100%; + z-index: 100 +} + +@media screen and (min-width: 30em) { + .swagger-ui .aspect-ratio-ns { + height:0; + position: relative + } + + .swagger-ui .aspect-ratio--16x9-ns { + padding-bottom: 56.25% + } + + .swagger-ui .aspect-ratio--9x16-ns { + padding-bottom: 177.77% + } + + .swagger-ui .aspect-ratio--4x3-ns { + padding-bottom: 75% + } + + .swagger-ui .aspect-ratio--3x4-ns { + padding-bottom: 133.33% + } + + .swagger-ui .aspect-ratio--6x4-ns { + padding-bottom: 66.6% + } + + .swagger-ui .aspect-ratio--4x6-ns { + padding-bottom: 150% + } + + .swagger-ui .aspect-ratio--8x5-ns { + padding-bottom: 62.5% + } + + .swagger-ui .aspect-ratio--5x8-ns { + padding-bottom: 160% + } + + .swagger-ui .aspect-ratio--7x5-ns { + padding-bottom: 71.42% + } + + .swagger-ui .aspect-ratio--5x7-ns { + padding-bottom: 140% + } + + .swagger-ui .aspect-ratio--1x1-ns { + padding-bottom: 100% + } + + .swagger-ui .aspect-ratio--object-ns { + bottom: 0; + height: 100%; + left: 0; + position: absolute; + right: 0; + top: 0; + width: 100%; + z-index: 100 + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .aspect-ratio-m { + height:0; + position: relative + } + + .swagger-ui .aspect-ratio--16x9-m { + padding-bottom: 56.25% + } + + .swagger-ui .aspect-ratio--9x16-m { + padding-bottom: 177.77% + } + + .swagger-ui .aspect-ratio--4x3-m { + padding-bottom: 75% + } + + .swagger-ui .aspect-ratio--3x4-m { + padding-bottom: 133.33% + } + + .swagger-ui .aspect-ratio--6x4-m { + padding-bottom: 66.6% + } + + .swagger-ui .aspect-ratio--4x6-m { + padding-bottom: 150% + } + + .swagger-ui .aspect-ratio--8x5-m { + padding-bottom: 62.5% + } + + .swagger-ui .aspect-ratio--5x8-m { + padding-bottom: 160% + } + + .swagger-ui .aspect-ratio--7x5-m { + padding-bottom: 71.42% + } + + .swagger-ui .aspect-ratio--5x7-m { + padding-bottom: 140% + } + + .swagger-ui .aspect-ratio--1x1-m { + padding-bottom: 100% + } + + .swagger-ui .aspect-ratio--object-m { + bottom: 0; + height: 100%; + left: 0; + position: absolute; + right: 0; + top: 0; + width: 100%; + z-index: 100 + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .aspect-ratio-l { + height:0; + position: relative + } + + .swagger-ui .aspect-ratio--16x9-l { + padding-bottom: 56.25% + } + + .swagger-ui .aspect-ratio--9x16-l { + padding-bottom: 177.77% + } + + .swagger-ui .aspect-ratio--4x3-l { + padding-bottom: 75% + } + + .swagger-ui .aspect-ratio--3x4-l { + padding-bottom: 133.33% + } + + .swagger-ui .aspect-ratio--6x4-l { + padding-bottom: 66.6% + } + + .swagger-ui .aspect-ratio--4x6-l { + padding-bottom: 150% + } + + .swagger-ui .aspect-ratio--8x5-l { + padding-bottom: 62.5% + } + + .swagger-ui .aspect-ratio--5x8-l { + padding-bottom: 160% + } + + .swagger-ui .aspect-ratio--7x5-l { + padding-bottom: 71.42% + } + + .swagger-ui .aspect-ratio--5x7-l { + padding-bottom: 140% + } + + .swagger-ui .aspect-ratio--1x1-l { + padding-bottom: 100% + } + + .swagger-ui .aspect-ratio--object-l { + bottom: 0; + height: 100%; + left: 0; + position: absolute; + right: 0; + top: 0; + width: 100%; + z-index: 100 + } +} + +.swagger-ui img { + max-width: 100% +} + +.swagger-ui .cover { + background-size: cover!important +} + +.swagger-ui .contain { + background-size: contain!important +} + +@media screen and (min-width: 30em) { + .swagger-ui .cover-ns { + background-size:cover!important + } + + .swagger-ui .contain-ns { + background-size: contain!important + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .cover-m { + background-size:cover!important + } + + .swagger-ui .contain-m { + background-size: contain!important + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .cover-l { + background-size:cover!important + } + + .swagger-ui .contain-l { + background-size: contain!important + } +} + +.swagger-ui .bg-center { + background-position: 50%; + background-repeat: no-repeat +} + +.swagger-ui .bg-top { + background-position: top; + background-repeat: no-repeat +} + +.swagger-ui .bg-right { + background-position: 100%; + background-repeat: no-repeat +} + +.swagger-ui .bg-bottom { + background-position: bottom; + background-repeat: no-repeat +} + +.swagger-ui .bg-left { + background-position: 0; + background-repeat: no-repeat +} + +@media screen and (min-width: 30em) { + .swagger-ui .bg-center-ns { + background-position:50%; + background-repeat: no-repeat + } + + .swagger-ui .bg-top-ns { + background-position: top; + background-repeat: no-repeat + } + + .swagger-ui .bg-right-ns { + background-position: 100%; + background-repeat: no-repeat + } + + .swagger-ui .bg-bottom-ns { + background-position: bottom; + background-repeat: no-repeat + } + + .swagger-ui .bg-left-ns { + background-position: 0; + background-repeat: no-repeat + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .bg-center-m { + background-position:50%; + background-repeat: no-repeat + } + + .swagger-ui .bg-top-m { + background-position: top; + background-repeat: no-repeat + } + + .swagger-ui .bg-right-m { + background-position: 100%; + background-repeat: no-repeat + } + + .swagger-ui .bg-bottom-m { + background-position: bottom; + background-repeat: no-repeat + } + + .swagger-ui .bg-left-m { + background-position: 0; + background-repeat: no-repeat + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .bg-center-l { + background-position:50%; + background-repeat: no-repeat + } + + .swagger-ui .bg-top-l { + background-position: top; + background-repeat: no-repeat + } + + .swagger-ui .bg-right-l { + background-position: 100%; + background-repeat: no-repeat + } + + .swagger-ui .bg-bottom-l { + background-position: bottom; + background-repeat: no-repeat + } + + .swagger-ui .bg-left-l { + background-position: 0; + background-repeat: no-repeat + } +} + +.swagger-ui .pre { + overflow-x: auto; + overflow-y: hidden; + overflow: scroll +} + +.swagger-ui .top-0 { + top: 0 +} + +.swagger-ui .right-0 { + right: 0 +} + +.swagger-ui .bottom-0 { + bottom: 0 +} + +.swagger-ui .left-0 { + left: 0 +} + +.swagger-ui .top-1 { + top: 1rem +} + +.swagger-ui .right-1 { + right: 1rem +} + +.swagger-ui .bottom-1 { + bottom: 1rem +} + +.swagger-ui .left-1 { + left: 1rem +} + +.swagger-ui .top-2 { + top: 2rem +} + +.swagger-ui .right-2 { + right: 2rem +} + +.swagger-ui .bottom-2 { + bottom: 2rem +} + +.swagger-ui .left-2 { + left: 2rem +} + +.swagger-ui .top--1 { + top: -1rem +} + +.swagger-ui .right--1 { + right: -1rem +} + +.swagger-ui .bottom--1 { + bottom: -1rem +} + +.swagger-ui .left--1 { + left: -1rem +} + +.swagger-ui .top--2 { + top: -2rem +} + +.swagger-ui .right--2 { + right: -2rem +} + +.swagger-ui .bottom--2 { + bottom: -2rem +} + +.swagger-ui .left--2 { + left: -2rem +} + +.swagger-ui .absolute--fill { + bottom: 0; + left: 0; + right: 0; + top: 0 +} + +@media screen and (min-width: 30em) { + .swagger-ui .top-0-ns { + top:0 + } + + .swagger-ui .left-0-ns { + left: 0 + } + + .swagger-ui .right-0-ns { + right: 0 + } + + .swagger-ui .bottom-0-ns { + bottom: 0 + } + + .swagger-ui .top-1-ns { + top: 1rem + } + + .swagger-ui .left-1-ns { + left: 1rem + } + + .swagger-ui .right-1-ns { + right: 1rem + } + + .swagger-ui .bottom-1-ns { + bottom: 1rem + } + + .swagger-ui .top-2-ns { + top: 2rem + } + + .swagger-ui .left-2-ns { + left: 2rem + } + + .swagger-ui .right-2-ns { + right: 2rem + } + + .swagger-ui .bottom-2-ns { + bottom: 2rem + } + + .swagger-ui .top--1-ns { + top: -1rem + } + + .swagger-ui .right--1-ns { + right: -1rem + } + + .swagger-ui .bottom--1-ns { + bottom: -1rem + } + + .swagger-ui .left--1-ns { + left: -1rem + } + + .swagger-ui .top--2-ns { + top: -2rem + } + + .swagger-ui .right--2-ns { + right: -2rem + } + + .swagger-ui .bottom--2-ns { + bottom: -2rem + } + + .swagger-ui .left--2-ns { + left: -2rem + } + + .swagger-ui .absolute--fill-ns { + bottom: 0; + left: 0; + right: 0; + top: 0 + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .top-0-m { + top:0 + } + + .swagger-ui .left-0-m { + left: 0 + } + + .swagger-ui .right-0-m { + right: 0 + } + + .swagger-ui .bottom-0-m { + bottom: 0 + } + + .swagger-ui .top-1-m { + top: 1rem + } + + .swagger-ui .left-1-m { + left: 1rem + } + + .swagger-ui .right-1-m { + right: 1rem + } + + .swagger-ui .bottom-1-m { + bottom: 1rem + } + + .swagger-ui .top-2-m { + top: 2rem + } + + .swagger-ui .left-2-m { + left: 2rem + } + + .swagger-ui .right-2-m { + right: 2rem + } + + .swagger-ui .bottom-2-m { + bottom: 2rem + } + + .swagger-ui .top--1-m { + top: -1rem + } + + .swagger-ui .right--1-m { + right: -1rem + } + + .swagger-ui .bottom--1-m { + bottom: -1rem + } + + .swagger-ui .left--1-m { + left: -1rem + } + + .swagger-ui .top--2-m { + top: -2rem + } + + .swagger-ui .right--2-m { + right: -2rem + } + + .swagger-ui .bottom--2-m { + bottom: -2rem + } + + .swagger-ui .left--2-m { + left: -2rem + } + + .swagger-ui .absolute--fill-m { + bottom: 0; + left: 0; + right: 0; + top: 0 + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .top-0-l { + top:0 + } + + .swagger-ui .left-0-l { + left: 0 + } + + .swagger-ui .right-0-l { + right: 0 + } + + .swagger-ui .bottom-0-l { + bottom: 0 + } + + .swagger-ui .top-1-l { + top: 1rem + } + + .swagger-ui .left-1-l { + left: 1rem + } + + .swagger-ui .right-1-l { + right: 1rem + } + + .swagger-ui .bottom-1-l { + bottom: 1rem + } + + .swagger-ui .top-2-l { + top: 2rem + } + + .swagger-ui .left-2-l { + left: 2rem + } + + .swagger-ui .right-2-l { + right: 2rem + } + + .swagger-ui .bottom-2-l { + bottom: 2rem + } + + .swagger-ui .top--1-l { + top: -1rem + } + + .swagger-ui .right--1-l { + right: -1rem + } + + .swagger-ui .bottom--1-l { + bottom: -1rem + } + + .swagger-ui .left--1-l { + left: -1rem + } + + .swagger-ui .top--2-l { + top: -2rem + } + + .swagger-ui .right--2-l { + right: -2rem + } + + .swagger-ui .bottom--2-l { + bottom: -2rem + } + + .swagger-ui .left--2-l { + left: -2rem + } + + .swagger-ui .absolute--fill-l { + bottom: 0; + left: 0; + right: 0; + top: 0 + } +} + +.swagger-ui .cf:after,.swagger-ui .cf:before { + content: " "; + display: table +} + +.swagger-ui .cf:after { + clear: both +} + +.swagger-ui .cf { + zoom:1} + +.swagger-ui .cl { + clear: left +} + +.swagger-ui .cr { + clear: right +} + +.swagger-ui .cb { + clear: both +} + +.swagger-ui .cn { + clear: none +} + +@media screen and (min-width: 30em) { + .swagger-ui .cl-ns { + clear:left + } + + .swagger-ui .cr-ns { + clear: right + } + + .swagger-ui .cb-ns { + clear: both + } + + .swagger-ui .cn-ns { + clear: none + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .cl-m { + clear:left + } + + .swagger-ui .cr-m { + clear: right + } + + .swagger-ui .cb-m { + clear: both + } + + .swagger-ui .cn-m { + clear: none + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .cl-l { + clear:left + } + + .swagger-ui .cr-l { + clear: right + } + + .swagger-ui .cb-l { + clear: both + } + + .swagger-ui .cn-l { + clear: none + } +} + +.swagger-ui .flex { + display: flex +} + +.swagger-ui .inline-flex { + display: inline-flex +} + +.swagger-ui .flex-auto { + flex: 1 1 auto; + min-height: 0; + min-width: 0 +} + +.swagger-ui .flex-none { + flex: none +} + +.swagger-ui .flex-column { + flex-direction: column +} + +.swagger-ui .flex-row { + flex-direction: row +} + +.swagger-ui .flex-wrap { + flex-wrap: wrap +} + +.swagger-ui .flex-nowrap { + flex-wrap: nowrap +} + +.swagger-ui .flex-wrap-reverse { + flex-wrap: wrap-reverse +} + +.swagger-ui .flex-column-reverse { + flex-direction: column-reverse +} + +.swagger-ui .flex-row-reverse { + flex-direction: row-reverse +} + +.swagger-ui .items-start { + align-items: flex-start +} + +.swagger-ui .items-end { + align-items: flex-end +} + +.swagger-ui .items-center { + align-items: center +} + +.swagger-ui .items-baseline { + align-items: baseline +} + +.swagger-ui .items-stretch { + align-items: stretch +} + +.swagger-ui .self-start { + align-self: flex-start +} + +.swagger-ui .self-end { + align-self: flex-end +} + +.swagger-ui .self-center { + align-self: center +} + +.swagger-ui .self-baseline { + align-self: baseline +} + +.swagger-ui .self-stretch { + align-self: stretch +} + +.swagger-ui .justify-start { + justify-content: flex-start +} + +.swagger-ui .justify-end { + justify-content: flex-end +} + +.swagger-ui .justify-center { + justify-content: center +} + +.swagger-ui .justify-between { + justify-content: space-between +} + +.swagger-ui .justify-around { + justify-content: space-around +} + +.swagger-ui .content-start { + align-content: flex-start +} + +.swagger-ui .content-end { + align-content: flex-end +} + +.swagger-ui .content-center { + align-content: center +} + +.swagger-ui .content-between { + align-content: space-between +} + +.swagger-ui .content-around { + align-content: space-around +} + +.swagger-ui .content-stretch { + align-content: stretch +} + +.swagger-ui .order-0 { + order: 0 +} + +.swagger-ui .order-1 { + order: 1 +} + +.swagger-ui .order-2 { + order: 2 +} + +.swagger-ui .order-3 { + order: 3 +} + +.swagger-ui .order-4 { + order: 4 +} + +.swagger-ui .order-5 { + order: 5 +} + +.swagger-ui .order-6 { + order: 6 +} + +.swagger-ui .order-7 { + order: 7 +} + +.swagger-ui .order-8 { + order: 8 +} + +.swagger-ui .order-last { + order: 99999 +} + +.swagger-ui .flex-grow-0 { + flex-grow: 0 +} + +.swagger-ui .flex-grow-1 { + flex-grow: 1 +} + +.swagger-ui .flex-shrink-0 { + flex-shrink: 0 +} + +.swagger-ui .flex-shrink-1 { + flex-shrink: 1 +} + +@media screen and (min-width: 30em) { + .swagger-ui .flex-ns { + display:flex + } + + .swagger-ui .inline-flex-ns { + display: inline-flex + } + + .swagger-ui .flex-auto-ns { + flex: 1 1 auto; + min-height: 0; + min-width: 0 + } + + .swagger-ui .flex-none-ns { + flex: none + } + + .swagger-ui .flex-column-ns { + flex-direction: column + } + + .swagger-ui .flex-row-ns { + flex-direction: row + } + + .swagger-ui .flex-wrap-ns { + flex-wrap: wrap + } + + .swagger-ui .flex-nowrap-ns { + flex-wrap: nowrap + } + + .swagger-ui .flex-wrap-reverse-ns { + flex-wrap: wrap-reverse + } + + .swagger-ui .flex-column-reverse-ns { + flex-direction: column-reverse + } + + .swagger-ui .flex-row-reverse-ns { + flex-direction: row-reverse + } + + .swagger-ui .items-start-ns { + align-items: flex-start + } + + .swagger-ui .items-end-ns { + align-items: flex-end + } + + .swagger-ui .items-center-ns { + align-items: center + } + + .swagger-ui .items-baseline-ns { + align-items: baseline + } + + .swagger-ui .items-stretch-ns { + align-items: stretch + } + + .swagger-ui .self-start-ns { + align-self: flex-start + } + + .swagger-ui .self-end-ns { + align-self: flex-end + } + + .swagger-ui .self-center-ns { + align-self: center + } + + .swagger-ui .self-baseline-ns { + align-self: baseline + } + + .swagger-ui .self-stretch-ns { + align-self: stretch + } + + .swagger-ui .justify-start-ns { + justify-content: flex-start + } + + .swagger-ui .justify-end-ns { + justify-content: flex-end + } + + .swagger-ui .justify-center-ns { + justify-content: center + } + + .swagger-ui .justify-between-ns { + justify-content: space-between + } + + .swagger-ui .justify-around-ns { + justify-content: space-around + } + + .swagger-ui .content-start-ns { + align-content: flex-start + } + + .swagger-ui .content-end-ns { + align-content: flex-end + } + + .swagger-ui .content-center-ns { + align-content: center + } + + .swagger-ui .content-between-ns { + align-content: space-between + } + + .swagger-ui .content-around-ns { + align-content: space-around + } + + .swagger-ui .content-stretch-ns { + align-content: stretch + } + + .swagger-ui .order-0-ns { + order: 0 + } + + .swagger-ui .order-1-ns { + order: 1 + } + + .swagger-ui .order-2-ns { + order: 2 + } + + .swagger-ui .order-3-ns { + order: 3 + } + + .swagger-ui .order-4-ns { + order: 4 + } + + .swagger-ui .order-5-ns { + order: 5 + } + + .swagger-ui .order-6-ns { + order: 6 + } + + .swagger-ui .order-7-ns { + order: 7 + } + + .swagger-ui .order-8-ns { + order: 8 + } + + .swagger-ui .order-last-ns { + order: 99999 + } + + .swagger-ui .flex-grow-0-ns { + flex-grow: 0 + } + + .swagger-ui .flex-grow-1-ns { + flex-grow: 1 + } + + .swagger-ui .flex-shrink-0-ns { + flex-shrink: 0 + } + + .swagger-ui .flex-shrink-1-ns { + flex-shrink: 1 + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .flex-m { + display:flex + } + + .swagger-ui .inline-flex-m { + display: inline-flex + } + + .swagger-ui .flex-auto-m { + flex: 1 1 auto; + min-height: 0; + min-width: 0 + } + + .swagger-ui .flex-none-m { + flex: none + } + + .swagger-ui .flex-column-m { + flex-direction: column + } + + .swagger-ui .flex-row-m { + flex-direction: row + } + + .swagger-ui .flex-wrap-m { + flex-wrap: wrap + } + + .swagger-ui .flex-nowrap-m { + flex-wrap: nowrap + } + + .swagger-ui .flex-wrap-reverse-m { + flex-wrap: wrap-reverse + } + + .swagger-ui .flex-column-reverse-m { + flex-direction: column-reverse + } + + .swagger-ui .flex-row-reverse-m { + flex-direction: row-reverse + } + + .swagger-ui .items-start-m { + align-items: flex-start + } + + .swagger-ui .items-end-m { + align-items: flex-end + } + + .swagger-ui .items-center-m { + align-items: center + } + + .swagger-ui .items-baseline-m { + align-items: baseline + } + + .swagger-ui .items-stretch-m { + align-items: stretch + } + + .swagger-ui .self-start-m { + align-self: flex-start + } + + .swagger-ui .self-end-m { + align-self: flex-end + } + + .swagger-ui .self-center-m { + align-self: center + } + + .swagger-ui .self-baseline-m { + align-self: baseline + } + + .swagger-ui .self-stretch-m { + align-self: stretch + } + + .swagger-ui .justify-start-m { + justify-content: flex-start + } + + .swagger-ui .justify-end-m { + justify-content: flex-end + } + + .swagger-ui .justify-center-m { + justify-content: center + } + + .swagger-ui .justify-between-m { + justify-content: space-between + } + + .swagger-ui .justify-around-m { + justify-content: space-around + } + + .swagger-ui .content-start-m { + align-content: flex-start + } + + .swagger-ui .content-end-m { + align-content: flex-end + } + + .swagger-ui .content-center-m { + align-content: center + } + + .swagger-ui .content-between-m { + align-content: space-between + } + + .swagger-ui .content-around-m { + align-content: space-around + } + + .swagger-ui .content-stretch-m { + align-content: stretch + } + + .swagger-ui .order-0-m { + order: 0 + } + + .swagger-ui .order-1-m { + order: 1 + } + + .swagger-ui .order-2-m { + order: 2 + } + + .swagger-ui .order-3-m { + order: 3 + } + + .swagger-ui .order-4-m { + order: 4 + } + + .swagger-ui .order-5-m { + order: 5 + } + + .swagger-ui .order-6-m { + order: 6 + } + + .swagger-ui .order-7-m { + order: 7 + } + + .swagger-ui .order-8-m { + order: 8 + } + + .swagger-ui .order-last-m { + order: 99999 + } + + .swagger-ui .flex-grow-0-m { + flex-grow: 0 + } + + .swagger-ui .flex-grow-1-m { + flex-grow: 1 + } + + .swagger-ui .flex-shrink-0-m { + flex-shrink: 0 + } + + .swagger-ui .flex-shrink-1-m { + flex-shrink: 1 + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .flex-l { + display:flex + } + + .swagger-ui .inline-flex-l { + display: inline-flex + } + + .swagger-ui .flex-auto-l { + flex: 1 1 auto; + min-height: 0; + min-width: 0 + } + + .swagger-ui .flex-none-l { + flex: none + } + + .swagger-ui .flex-column-l { + flex-direction: column + } + + .swagger-ui .flex-row-l { + flex-direction: row + } + + .swagger-ui .flex-wrap-l { + flex-wrap: wrap + } + + .swagger-ui .flex-nowrap-l { + flex-wrap: nowrap + } + + .swagger-ui .flex-wrap-reverse-l { + flex-wrap: wrap-reverse + } + + .swagger-ui .flex-column-reverse-l { + flex-direction: column-reverse + } + + .swagger-ui .flex-row-reverse-l { + flex-direction: row-reverse + } + + .swagger-ui .items-start-l { + align-items: flex-start + } + + .swagger-ui .items-end-l { + align-items: flex-end + } + + .swagger-ui .items-center-l { + align-items: center + } + + .swagger-ui .items-baseline-l { + align-items: baseline + } + + .swagger-ui .items-stretch-l { + align-items: stretch + } + + .swagger-ui .self-start-l { + align-self: flex-start + } + + .swagger-ui .self-end-l { + align-self: flex-end + } + + .swagger-ui .self-center-l { + align-self: center + } + + .swagger-ui .self-baseline-l { + align-self: baseline + } + + .swagger-ui .self-stretch-l { + align-self: stretch + } + + .swagger-ui .justify-start-l { + justify-content: flex-start + } + + .swagger-ui .justify-end-l { + justify-content: flex-end + } + + .swagger-ui .justify-center-l { + justify-content: center + } + + .swagger-ui .justify-between-l { + justify-content: space-between + } + + .swagger-ui .justify-around-l { + justify-content: space-around + } + + .swagger-ui .content-start-l { + align-content: flex-start + } + + .swagger-ui .content-end-l { + align-content: flex-end + } + + .swagger-ui .content-center-l { + align-content: center + } + + .swagger-ui .content-between-l { + align-content: space-between + } + + .swagger-ui .content-around-l { + align-content: space-around + } + + .swagger-ui .content-stretch-l { + align-content: stretch + } + + .swagger-ui .order-0-l { + order: 0 + } + + .swagger-ui .order-1-l { + order: 1 + } + + .swagger-ui .order-2-l { + order: 2 + } + + .swagger-ui .order-3-l { + order: 3 + } + + .swagger-ui .order-4-l { + order: 4 + } + + .swagger-ui .order-5-l { + order: 5 + } + + .swagger-ui .order-6-l { + order: 6 + } + + .swagger-ui .order-7-l { + order: 7 + } + + .swagger-ui .order-8-l { + order: 8 + } + + .swagger-ui .order-last-l { + order: 99999 + } + + .swagger-ui .flex-grow-0-l { + flex-grow: 0 + } + + .swagger-ui .flex-grow-1-l { + flex-grow: 1 + } + + .swagger-ui .flex-shrink-0-l { + flex-shrink: 0 + } + + .swagger-ui .flex-shrink-1-l { + flex-shrink: 1 + } +} + +.swagger-ui .dn { + display: none +} + +.swagger-ui .di { + display: inline +} + +.swagger-ui .db { + display: block +} + +.swagger-ui .dib { + display: inline-block +} + +.swagger-ui .dit { + display: inline-table +} + +.swagger-ui .dt { + display: table +} + +.swagger-ui .dtc { + display: table-cell +} + +.swagger-ui .dt-row { + display: table-row +} + +.swagger-ui .dt-row-group { + display: table-row-group +} + +.swagger-ui .dt-column { + display: table-column +} + +.swagger-ui .dt-column-group { + display: table-column-group +} + +.swagger-ui .dt--fixed { + table-layout: fixed; + width: 100% +} + +@media screen and (min-width: 30em) { + .swagger-ui .dn-ns { + display:none + } + + .swagger-ui .di-ns { + display: inline + } + + .swagger-ui .db-ns { + display: block + } + + .swagger-ui .dib-ns { + display: inline-block + } + + .swagger-ui .dit-ns { + display: inline-table + } + + .swagger-ui .dt-ns { + display: table + } + + .swagger-ui .dtc-ns { + display: table-cell + } + + .swagger-ui .dt-row-ns { + display: table-row + } + + .swagger-ui .dt-row-group-ns { + display: table-row-group + } + + .swagger-ui .dt-column-ns { + display: table-column + } + + .swagger-ui .dt-column-group-ns { + display: table-column-group + } + + .swagger-ui .dt--fixed-ns { + table-layout: fixed; + width: 100% + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .dn-m { + display:none + } + + .swagger-ui .di-m { + display: inline + } + + .swagger-ui .db-m { + display: block + } + + .swagger-ui .dib-m { + display: inline-block + } + + .swagger-ui .dit-m { + display: inline-table + } + + .swagger-ui .dt-m { + display: table + } + + .swagger-ui .dtc-m { + display: table-cell + } + + .swagger-ui .dt-row-m { + display: table-row + } + + .swagger-ui .dt-row-group-m { + display: table-row-group + } + + .swagger-ui .dt-column-m { + display: table-column + } + + .swagger-ui .dt-column-group-m { + display: table-column-group + } + + .swagger-ui .dt--fixed-m { + table-layout: fixed; + width: 100% + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .dn-l { + display:none + } + + .swagger-ui .di-l { + display: inline + } + + .swagger-ui .db-l { + display: block + } + + .swagger-ui .dib-l { + display: inline-block + } + + .swagger-ui .dit-l { + display: inline-table + } + + .swagger-ui .dt-l { + display: table + } + + .swagger-ui .dtc-l { + display: table-cell + } + + .swagger-ui .dt-row-l { + display: table-row + } + + .swagger-ui .dt-row-group-l { + display: table-row-group + } + + .swagger-ui .dt-column-l { + display: table-column + } + + .swagger-ui .dt-column-group-l { + display: table-column-group + } + + .swagger-ui .dt--fixed-l { + table-layout: fixed; + width: 100% + } +} + +.swagger-ui .fl { + _display: inline; + float: left +} + +.swagger-ui .fr { + _display: inline; + float: right +} + +.swagger-ui .fn { + float: none +} + +@media screen and (min-width: 30em) { + .swagger-ui .fl-ns { + _display:inline; + float: left + } + + .swagger-ui .fr-ns { + _display: inline; + float: right + } + + .swagger-ui .fn-ns { + float: none + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .fl-m { + _display:inline; + float: left + } + + .swagger-ui .fr-m { + _display: inline; + float: right + } + + .swagger-ui .fn-m { + float: none + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .fl-l { + _display:inline; + float: left + } + + .swagger-ui .fr-l { + _display: inline; + float: right + } + + .swagger-ui .fn-l { + float: none + } +} + +.swagger-ui .sans-serif { + font-family: -apple-system,BlinkMacSystemFont,avenir next,avenir,helvetica,helvetica neue,ubuntu,roboto,noto,segoe ui,arial,sans-serif +} + +.swagger-ui .serif { + font-family: georgia,serif +} + +.swagger-ui .system-sans-serif { + font-family: sans-serif +} + +.swagger-ui .system-serif { + font-family: serif +} + +.swagger-ui .code,.swagger-ui code { + font-family: Consolas,monaco,monospace +} + +.swagger-ui .courier { + font-family: Courier Next,courier,monospace +} + +.swagger-ui .helvetica { + font-family: helvetica neue,helvetica,sans-serif +} + +.swagger-ui .avenir { + font-family: avenir next,avenir,sans-serif +} + +.swagger-ui .athelas { + font-family: athelas,georgia,serif +} + +.swagger-ui .georgia { + font-family: georgia,serif +} + +.swagger-ui .times { + font-family: times,serif +} + +.swagger-ui .bodoni { + font-family: Bodoni MT,serif +} + +.swagger-ui .calisto { + font-family: Calisto MT,serif +} + +.swagger-ui .garamond { + font-family: garamond,serif +} + +.swagger-ui .baskerville { + font-family: baskerville,serif +} + +.swagger-ui .i { + font-style: italic +} + +.swagger-ui .fs-normal { + font-style: normal +} + +@media screen and (min-width: 30em) { + .swagger-ui .i-ns { + font-style:italic + } + + .swagger-ui .fs-normal-ns { + font-style: normal + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .i-m { + font-style:italic + } + + .swagger-ui .fs-normal-m { + font-style: normal + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .i-l { + font-style:italic + } + + .swagger-ui .fs-normal-l { + font-style: normal + } +} + +.swagger-ui .normal { + font-weight: 400 +} + +.swagger-ui .b { + font-weight: 700 +} + +.swagger-ui .fw1 { + font-weight: 100 +} + +.swagger-ui .fw2 { + font-weight: 200 +} + +.swagger-ui .fw3 { + font-weight: 300 +} + +.swagger-ui .fw4 { + font-weight: 400 +} + +.swagger-ui .fw5 { + font-weight: 500 +} + +.swagger-ui .fw6 { + font-weight: 600 +} + +.swagger-ui .fw7 { + font-weight: 700 +} + +.swagger-ui .fw8 { + font-weight: 800 +} + +.swagger-ui .fw9 { + font-weight: 900 +} + +@media screen and (min-width: 30em) { + .swagger-ui .normal-ns { + font-weight:400 + } + + .swagger-ui .b-ns { + font-weight: 700 + } + + .swagger-ui .fw1-ns { + font-weight: 100 + } + + .swagger-ui .fw2-ns { + font-weight: 200 + } + + .swagger-ui .fw3-ns { + font-weight: 300 + } + + .swagger-ui .fw4-ns { + font-weight: 400 + } + + .swagger-ui .fw5-ns { + font-weight: 500 + } + + .swagger-ui .fw6-ns { + font-weight: 600 + } + + .swagger-ui .fw7-ns { + font-weight: 700 + } + + .swagger-ui .fw8-ns { + font-weight: 800 + } + + .swagger-ui .fw9-ns { + font-weight: 900 + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .normal-m { + font-weight:400 + } + + .swagger-ui .b-m { + font-weight: 700 + } + + .swagger-ui .fw1-m { + font-weight: 100 + } + + .swagger-ui .fw2-m { + font-weight: 200 + } + + .swagger-ui .fw3-m { + font-weight: 300 + } + + .swagger-ui .fw4-m { + font-weight: 400 + } + + .swagger-ui .fw5-m { + font-weight: 500 + } + + .swagger-ui .fw6-m { + font-weight: 600 + } + + .swagger-ui .fw7-m { + font-weight: 700 + } + + .swagger-ui .fw8-m { + font-weight: 800 + } + + .swagger-ui .fw9-m { + font-weight: 900 + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .normal-l { + font-weight:400 + } + + .swagger-ui .b-l { + font-weight: 700 + } + + .swagger-ui .fw1-l { + font-weight: 100 + } + + .swagger-ui .fw2-l { + font-weight: 200 + } + + .swagger-ui .fw3-l { + font-weight: 300 + } + + .swagger-ui .fw4-l { + font-weight: 400 + } + + .swagger-ui .fw5-l { + font-weight: 500 + } + + .swagger-ui .fw6-l { + font-weight: 600 + } + + .swagger-ui .fw7-l { + font-weight: 700 + } + + .swagger-ui .fw8-l { + font-weight: 800 + } + + .swagger-ui .fw9-l { + font-weight: 900 + } +} + +.swagger-ui .input-reset { + -webkit-appearance: none; + -moz-appearance: none +} + +.swagger-ui .button-reset::-moz-focus-inner,.swagger-ui .input-reset::-moz-focus-inner { + padding: 0 +} + +.swagger-ui .h1 { + height: 1rem +} + +.swagger-ui .h2 { + height: 2rem +} + +.swagger-ui .h3 { + height: 4rem +} + +.swagger-ui .h4 { + height: 8rem +} + +.swagger-ui .h5 { + height: 16rem +} + +.swagger-ui .h-25 { + height: 25% +} + +.swagger-ui .h-50 { + height: 50% +} + +.swagger-ui .h-75 { + height: 75% +} + +.swagger-ui .h-100 { + height: 100% +} + +.swagger-ui .min-h-100 { + min-height: 100% +} + +.swagger-ui .vh-25 { + height: 25vh +} + +.swagger-ui .vh-50 { + height: 50vh +} + +.swagger-ui .vh-75 { + height: 75vh +} + +.swagger-ui .vh-100 { + height: 100vh +} + +.swagger-ui .min-vh-100 { + min-height: 100vh +} + +.swagger-ui .h-auto { + height: auto +} + +.swagger-ui .h-inherit { + height: inherit +} + +@media screen and (min-width: 30em) { + .swagger-ui .h1-ns { + height:1rem + } + + .swagger-ui .h2-ns { + height: 2rem + } + + .swagger-ui .h3-ns { + height: 4rem + } + + .swagger-ui .h4-ns { + height: 8rem + } + + .swagger-ui .h5-ns { + height: 16rem + } + + .swagger-ui .h-25-ns { + height: 25% + } + + .swagger-ui .h-50-ns { + height: 50% + } + + .swagger-ui .h-75-ns { + height: 75% + } + + .swagger-ui .h-100-ns { + height: 100% + } + + .swagger-ui .min-h-100-ns { + min-height: 100% + } + + .swagger-ui .vh-25-ns { + height: 25vh + } + + .swagger-ui .vh-50-ns { + height: 50vh + } + + .swagger-ui .vh-75-ns { + height: 75vh + } + + .swagger-ui .vh-100-ns { + height: 100vh + } + + .swagger-ui .min-vh-100-ns { + min-height: 100vh + } + + .swagger-ui .h-auto-ns { + height: auto + } + + .swagger-ui .h-inherit-ns { + height: inherit + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .h1-m { + height:1rem + } + + .swagger-ui .h2-m { + height: 2rem + } + + .swagger-ui .h3-m { + height: 4rem + } + + .swagger-ui .h4-m { + height: 8rem + } + + .swagger-ui .h5-m { + height: 16rem + } + + .swagger-ui .h-25-m { + height: 25% + } + + .swagger-ui .h-50-m { + height: 50% + } + + .swagger-ui .h-75-m { + height: 75% + } + + .swagger-ui .h-100-m { + height: 100% + } + + .swagger-ui .min-h-100-m { + min-height: 100% + } + + .swagger-ui .vh-25-m { + height: 25vh + } + + .swagger-ui .vh-50-m { + height: 50vh + } + + .swagger-ui .vh-75-m { + height: 75vh + } + + .swagger-ui .vh-100-m { + height: 100vh + } + + .swagger-ui .min-vh-100-m { + min-height: 100vh + } + + .swagger-ui .h-auto-m { + height: auto + } + + .swagger-ui .h-inherit-m { + height: inherit + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .h1-l { + height:1rem + } + + .swagger-ui .h2-l { + height: 2rem + } + + .swagger-ui .h3-l { + height: 4rem + } + + .swagger-ui .h4-l { + height: 8rem + } + + .swagger-ui .h5-l { + height: 16rem + } + + .swagger-ui .h-25-l { + height: 25% + } + + .swagger-ui .h-50-l { + height: 50% + } + + .swagger-ui .h-75-l { + height: 75% + } + + .swagger-ui .h-100-l { + height: 100% + } + + .swagger-ui .min-h-100-l { + min-height: 100% + } + + .swagger-ui .vh-25-l { + height: 25vh + } + + .swagger-ui .vh-50-l { + height: 50vh + } + + .swagger-ui .vh-75-l { + height: 75vh + } + + .swagger-ui .vh-100-l { + height: 100vh + } + + .swagger-ui .min-vh-100-l { + min-height: 100vh + } + + .swagger-ui .h-auto-l { + height: auto + } + + .swagger-ui .h-inherit-l { + height: inherit + } +} + +.swagger-ui .tracked { + letter-spacing: .1em +} + +.swagger-ui .tracked-tight { + letter-spacing: -.05em +} + +.swagger-ui .tracked-mega { + letter-spacing: .25em +} + +@media screen and (min-width: 30em) { + .swagger-ui .tracked-ns { + letter-spacing:.1em + } + + .swagger-ui .tracked-tight-ns { + letter-spacing: -.05em + } + + .swagger-ui .tracked-mega-ns { + letter-spacing: .25em + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .tracked-m { + letter-spacing:.1em + } + + .swagger-ui .tracked-tight-m { + letter-spacing: -.05em + } + + .swagger-ui .tracked-mega-m { + letter-spacing: .25em + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .tracked-l { + letter-spacing:.1em + } + + .swagger-ui .tracked-tight-l { + letter-spacing: -.05em + } + + .swagger-ui .tracked-mega-l { + letter-spacing: .25em + } +} + +.swagger-ui .lh-solid { + line-height: 1 +} + +.swagger-ui .lh-title { + line-height: 1.25 +} + +.swagger-ui .lh-copy { + line-height: 1.5 +} + +@media screen and (min-width: 30em) { + .swagger-ui .lh-solid-ns { + line-height:1 + } + + .swagger-ui .lh-title-ns { + line-height: 1.25 + } + + .swagger-ui .lh-copy-ns { + line-height: 1.5 + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .lh-solid-m { + line-height:1 + } + + .swagger-ui .lh-title-m { + line-height: 1.25 + } + + .swagger-ui .lh-copy-m { + line-height: 1.5 + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .lh-solid-l { + line-height:1 + } + + .swagger-ui .lh-title-l { + line-height: 1.25 + } + + .swagger-ui .lh-copy-l { + line-height: 1.5 + } +} + +.swagger-ui .link { + -webkit-text-decoration: none; + text-decoration: none +} + +.swagger-ui .link,.swagger-ui .link:active,.swagger-ui .link:focus,.swagger-ui .link:hover,.swagger-ui .link:link,.swagger-ui .link:visited { + transition: color .15s ease-in +} + +.swagger-ui .link:focus { + outline: 1px dotted currentColor +} + +.swagger-ui .list { + list-style-type: none +} + +.swagger-ui .mw-100 { + max-width: 100% +} + +.swagger-ui .mw1 { + max-width: 1rem +} + +.swagger-ui .mw2 { + max-width: 2rem +} + +.swagger-ui .mw3 { + max-width: 4rem +} + +.swagger-ui .mw4 { + max-width: 8rem +} + +.swagger-ui .mw5 { + max-width: 16rem +} + +.swagger-ui .mw6 { + max-width: 32rem +} + +.swagger-ui .mw7 { + max-width: 48rem +} + +.swagger-ui .mw8 { + max-width: 64rem +} + +.swagger-ui .mw9 { + max-width: 96rem +} + +.swagger-ui .mw-none { + max-width: none +} + +@media screen and (min-width: 30em) { + .swagger-ui .mw-100-ns { + max-width:100% + } + + .swagger-ui .mw1-ns { + max-width: 1rem + } + + .swagger-ui .mw2-ns { + max-width: 2rem + } + + .swagger-ui .mw3-ns { + max-width: 4rem + } + + .swagger-ui .mw4-ns { + max-width: 8rem + } + + .swagger-ui .mw5-ns { + max-width: 16rem + } + + .swagger-ui .mw6-ns { + max-width: 32rem + } + + .swagger-ui .mw7-ns { + max-width: 48rem + } + + .swagger-ui .mw8-ns { + max-width: 64rem + } + + .swagger-ui .mw9-ns { + max-width: 96rem + } + + .swagger-ui .mw-none-ns { + max-width: none + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .mw-100-m { + max-width:100% + } + + .swagger-ui .mw1-m { + max-width: 1rem + } + + .swagger-ui .mw2-m { + max-width: 2rem + } + + .swagger-ui .mw3-m { + max-width: 4rem + } + + .swagger-ui .mw4-m { + max-width: 8rem + } + + .swagger-ui .mw5-m { + max-width: 16rem + } + + .swagger-ui .mw6-m { + max-width: 32rem + } + + .swagger-ui .mw7-m { + max-width: 48rem + } + + .swagger-ui .mw8-m { + max-width: 64rem + } + + .swagger-ui .mw9-m { + max-width: 96rem + } + + .swagger-ui .mw-none-m { + max-width: none + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .mw-100-l { + max-width:100% + } + + .swagger-ui .mw1-l { + max-width: 1rem + } + + .swagger-ui .mw2-l { + max-width: 2rem + } + + .swagger-ui .mw3-l { + max-width: 4rem + } + + .swagger-ui .mw4-l { + max-width: 8rem + } + + .swagger-ui .mw5-l { + max-width: 16rem + } + + .swagger-ui .mw6-l { + max-width: 32rem + } + + .swagger-ui .mw7-l { + max-width: 48rem + } + + .swagger-ui .mw8-l { + max-width: 64rem + } + + .swagger-ui .mw9-l { + max-width: 96rem + } + + .swagger-ui .mw-none-l { + max-width: none + } +} + +.swagger-ui .w1 { + width: 1rem +} + +.swagger-ui .w2 { + width: 2rem +} + +.swagger-ui .w3 { + width: 4rem +} + +.swagger-ui .w4 { + width: 8rem +} + +.swagger-ui .w5 { + width: 16rem +} + +.swagger-ui .w-10 { + width: 10% +} + +.swagger-ui .w-20 { + width: 20% +} + +.swagger-ui .w-25 { + width: 25% +} + +.swagger-ui .w-30 { + width: 30% +} + +.swagger-ui .w-33 { + width: 33% +} + +.swagger-ui .w-34 { + width: 34% +} + +.swagger-ui .w-40 { + width: 40% +} + +.swagger-ui .w-50 { + width: 50% +} + +.swagger-ui .w-60 { + width: 60% +} + +.swagger-ui .w-70 { + width: 70% +} + +.swagger-ui .w-75 { + width: 75% +} + +.swagger-ui .w-80 { + width: 80% +} + +.swagger-ui .w-90 { + width: 90% +} + +.swagger-ui .w-100 { + width: 100% +} + +.swagger-ui .w-third { + width: 33.3333333333% +} + +.swagger-ui .w-two-thirds { + width: 66.6666666667% +} + +.swagger-ui .w-auto { + width: auto +} + +@media screen and (min-width: 30em) { + .swagger-ui .w1-ns { + width:1rem + } + + .swagger-ui .w2-ns { + width: 2rem + } + + .swagger-ui .w3-ns { + width: 4rem + } + + .swagger-ui .w4-ns { + width: 8rem + } + + .swagger-ui .w5-ns { + width: 16rem + } + + .swagger-ui .w-10-ns { + width: 10% + } + + .swagger-ui .w-20-ns { + width: 20% + } + + .swagger-ui .w-25-ns { + width: 25% + } + + .swagger-ui .w-30-ns { + width: 30% + } + + .swagger-ui .w-33-ns { + width: 33% + } + + .swagger-ui .w-34-ns { + width: 34% + } + + .swagger-ui .w-40-ns { + width: 40% + } + + .swagger-ui .w-50-ns { + width: 50% + } + + .swagger-ui .w-60-ns { + width: 60% + } + + .swagger-ui .w-70-ns { + width: 70% + } + + .swagger-ui .w-75-ns { + width: 75% + } + + .swagger-ui .w-80-ns { + width: 80% + } + + .swagger-ui .w-90-ns { + width: 90% + } + + .swagger-ui .w-100-ns { + width: 100% + } + + .swagger-ui .w-third-ns { + width: 33.3333333333% + } + + .swagger-ui .w-two-thirds-ns { + width: 66.6666666667% + } + + .swagger-ui .w-auto-ns { + width: auto + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .w1-m { + width:1rem + } + + .swagger-ui .w2-m { + width: 2rem + } + + .swagger-ui .w3-m { + width: 4rem + } + + .swagger-ui .w4-m { + width: 8rem + } + + .swagger-ui .w5-m { + width: 16rem + } + + .swagger-ui .w-10-m { + width: 10% + } + + .swagger-ui .w-20-m { + width: 20% + } + + .swagger-ui .w-25-m { + width: 25% + } + + .swagger-ui .w-30-m { + width: 30% + } + + .swagger-ui .w-33-m { + width: 33% + } + + .swagger-ui .w-34-m { + width: 34% + } + + .swagger-ui .w-40-m { + width: 40% + } + + .swagger-ui .w-50-m { + width: 50% + } + + .swagger-ui .w-60-m { + width: 60% + } + + .swagger-ui .w-70-m { + width: 70% + } + + .swagger-ui .w-75-m { + width: 75% + } + + .swagger-ui .w-80-m { + width: 80% + } + + .swagger-ui .w-90-m { + width: 90% + } + + .swagger-ui .w-100-m { + width: 100% + } + + .swagger-ui .w-third-m { + width: 33.3333333333% + } + + .swagger-ui .w-two-thirds-m { + width: 66.6666666667% + } + + .swagger-ui .w-auto-m { + width: auto + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .w1-l { + width:1rem + } + + .swagger-ui .w2-l { + width: 2rem + } + + .swagger-ui .w3-l { + width: 4rem + } + + .swagger-ui .w4-l { + width: 8rem + } + + .swagger-ui .w5-l { + width: 16rem + } + + .swagger-ui .w-10-l { + width: 10% + } + + .swagger-ui .w-20-l { + width: 20% + } + + .swagger-ui .w-25-l { + width: 25% + } + + .swagger-ui .w-30-l { + width: 30% + } + + .swagger-ui .w-33-l { + width: 33% + } + + .swagger-ui .w-34-l { + width: 34% + } + + .swagger-ui .w-40-l { + width: 40% + } + + .swagger-ui .w-50-l { + width: 50% + } + + .swagger-ui .w-60-l { + width: 60% + } + + .swagger-ui .w-70-l { + width: 70% + } + + .swagger-ui .w-75-l { + width: 75% + } + + .swagger-ui .w-80-l { + width: 80% + } + + .swagger-ui .w-90-l { + width: 90% + } + + .swagger-ui .w-100-l { + width: 100% + } + + .swagger-ui .w-third-l { + width: 33.3333333333% + } + + .swagger-ui .w-two-thirds-l { + width: 66.6666666667% + } + + .swagger-ui .w-auto-l { + width: auto + } +} + +.swagger-ui .overflow-visible { + overflow: visible +} + +.swagger-ui .overflow-hidden { + overflow: hidden +} + +.swagger-ui .overflow-scroll { + overflow: scroll +} + +.swagger-ui .overflow-auto { + overflow: auto +} + +.swagger-ui .overflow-x-visible { + overflow-x: visible +} + +.swagger-ui .overflow-x-hidden { + overflow-x: hidden +} + +.swagger-ui .overflow-x-scroll { + overflow-x: scroll +} + +.swagger-ui .overflow-x-auto { + overflow-x: auto +} + +.swagger-ui .overflow-y-visible { + overflow-y: visible +} + +.swagger-ui .overflow-y-hidden { + overflow-y: hidden +} + +.swagger-ui .overflow-y-scroll { + overflow-y: scroll +} + +.swagger-ui .overflow-y-auto { + overflow-y: auto +} + +@media screen and (min-width: 30em) { + .swagger-ui .overflow-visible-ns { + overflow:visible + } + + .swagger-ui .overflow-hidden-ns { + overflow: hidden + } + + .swagger-ui .overflow-scroll-ns { + overflow: scroll + } + + .swagger-ui .overflow-auto-ns { + overflow: auto + } + + .swagger-ui .overflow-x-visible-ns { + overflow-x: visible + } + + .swagger-ui .overflow-x-hidden-ns { + overflow-x: hidden + } + + .swagger-ui .overflow-x-scroll-ns { + overflow-x: scroll + } + + .swagger-ui .overflow-x-auto-ns { + overflow-x: auto + } + + .swagger-ui .overflow-y-visible-ns { + overflow-y: visible + } + + .swagger-ui .overflow-y-hidden-ns { + overflow-y: hidden + } + + .swagger-ui .overflow-y-scroll-ns { + overflow-y: scroll + } + + .swagger-ui .overflow-y-auto-ns { + overflow-y: auto + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .overflow-visible-m { + overflow:visible + } + + .swagger-ui .overflow-hidden-m { + overflow: hidden + } + + .swagger-ui .overflow-scroll-m { + overflow: scroll + } + + .swagger-ui .overflow-auto-m { + overflow: auto + } + + .swagger-ui .overflow-x-visible-m { + overflow-x: visible + } + + .swagger-ui .overflow-x-hidden-m { + overflow-x: hidden + } + + .swagger-ui .overflow-x-scroll-m { + overflow-x: scroll + } + + .swagger-ui .overflow-x-auto-m { + overflow-x: auto + } + + .swagger-ui .overflow-y-visible-m { + overflow-y: visible + } + + .swagger-ui .overflow-y-hidden-m { + overflow-y: hidden + } + + .swagger-ui .overflow-y-scroll-m { + overflow-y: scroll + } + + .swagger-ui .overflow-y-auto-m { + overflow-y: auto + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .overflow-visible-l { + overflow:visible + } + + .swagger-ui .overflow-hidden-l { + overflow: hidden + } + + .swagger-ui .overflow-scroll-l { + overflow: scroll + } + + .swagger-ui .overflow-auto-l { + overflow: auto + } + + .swagger-ui .overflow-x-visible-l { + overflow-x: visible + } + + .swagger-ui .overflow-x-hidden-l { + overflow-x: hidden + } + + .swagger-ui .overflow-x-scroll-l { + overflow-x: scroll + } + + .swagger-ui .overflow-x-auto-l { + overflow-x: auto + } + + .swagger-ui .overflow-y-visible-l { + overflow-y: visible + } + + .swagger-ui .overflow-y-hidden-l { + overflow-y: hidden + } + + .swagger-ui .overflow-y-scroll-l { + overflow-y: scroll + } + + .swagger-ui .overflow-y-auto-l { + overflow-y: auto + } +} + +.swagger-ui .static { + position: static +} + +.swagger-ui .relative { + position: relative +} + +.swagger-ui .absolute { + position: absolute +} + +.swagger-ui .fixed { + position: fixed +} + +@media screen and (min-width: 30em) { + .swagger-ui .static-ns { + position:static + } + + .swagger-ui .relative-ns { + position: relative + } + + .swagger-ui .absolute-ns { + position: absolute + } + + .swagger-ui .fixed-ns { + position: fixed + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .static-m { + position:static + } + + .swagger-ui .relative-m { + position: relative + } + + .swagger-ui .absolute-m { + position: absolute + } + + .swagger-ui .fixed-m { + position: fixed + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .static-l { + position:static + } + + .swagger-ui .relative-l { + position: relative + } + + .swagger-ui .absolute-l { + position: absolute + } + + .swagger-ui .fixed-l { + position: fixed + } +} + +.swagger-ui .o-100 { + opacity: 1 +} + +.swagger-ui .o-90 { + opacity: .9 +} + +.swagger-ui .o-80 { + opacity: .8 +} + +.swagger-ui .o-70 { + opacity: .7 +} + +.swagger-ui .o-60 { + opacity: .6 +} + +.swagger-ui .o-50 { + opacity: .5 +} + +.swagger-ui .o-40 { + opacity: .4 +} + +.swagger-ui .o-30 { + opacity: .3 +} + +.swagger-ui .o-20 { + opacity: .2 +} + +.swagger-ui .o-10 { + opacity: .1 +} + +.swagger-ui .o-05 { + opacity: .05 +} + +.swagger-ui .o-025 { + opacity: .025 +} + +.swagger-ui .o-0 { + opacity: 0 +} + +.swagger-ui .rotate-45 { + transform: rotate(45deg) +} + +.swagger-ui .rotate-90 { + transform: rotate(90deg) +} + +.swagger-ui .rotate-135 { + transform: rotate(135deg) +} + +.swagger-ui .rotate-180 { + transform: rotate(180deg) +} + +.swagger-ui .rotate-225 { + transform: rotate(225deg) +} + +.swagger-ui .rotate-270 { + transform: rotate(270deg) +} + +.swagger-ui .rotate-315 { + transform: rotate(315deg) +} + +@media screen and (min-width: 30em) { + .swagger-ui .rotate-45-ns { + transform:rotate(45deg) + } + + .swagger-ui .rotate-90-ns { + transform: rotate(90deg) + } + + .swagger-ui .rotate-135-ns { + transform: rotate(135deg) + } + + .swagger-ui .rotate-180-ns { + transform: rotate(180deg) + } + + .swagger-ui .rotate-225-ns { + transform: rotate(225deg) + } + + .swagger-ui .rotate-270-ns { + transform: rotate(270deg) + } + + .swagger-ui .rotate-315-ns { + transform: rotate(315deg) + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .rotate-45-m { + transform:rotate(45deg) + } + + .swagger-ui .rotate-90-m { + transform: rotate(90deg) + } + + .swagger-ui .rotate-135-m { + transform: rotate(135deg) + } + + .swagger-ui .rotate-180-m { + transform: rotate(180deg) + } + + .swagger-ui .rotate-225-m { + transform: rotate(225deg) + } + + .swagger-ui .rotate-270-m { + transform: rotate(270deg) + } + + .swagger-ui .rotate-315-m { + transform: rotate(315deg) + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .rotate-45-l { + transform:rotate(45deg) + } + + .swagger-ui .rotate-90-l { + transform: rotate(90deg) + } + + .swagger-ui .rotate-135-l { + transform: rotate(135deg) + } + + .swagger-ui .rotate-180-l { + transform: rotate(180deg) + } + + .swagger-ui .rotate-225-l { + transform: rotate(225deg) + } + + .swagger-ui .rotate-270-l { + transform: rotate(270deg) + } + + .swagger-ui .rotate-315-l { + transform: rotate(315deg) + } +} + +.swagger-ui .black-90 { + color: rgba(0,0,0,.9) +} + +.swagger-ui .black-80 { + color: rgba(0,0,0,.8) +} + +.swagger-ui .black-70 { + color: rgba(0,0,0,.7) +} + +.swagger-ui .black-60 { + color: rgba(0,0,0,.6) +} + +.swagger-ui .black-50 { + color: rgba(0,0,0,.5) +} + +.swagger-ui .black-40 { + color: rgba(0,0,0,.4) +} + +.swagger-ui .black-30 { + color: rgba(0,0,0,.3) +} + +.swagger-ui .black-20 { + color: rgba(0,0,0,.2) +} + +.swagger-ui .black-10 { + color: rgba(0,0,0,.1) +} + +.swagger-ui .black-05 { + color: rgba(0,0,0,.05) +} + +.swagger-ui .white-90 { + color: hsla(0,0%,100%,.9) +} + +.swagger-ui .white-80 { + color: hsla(0,0%,100%,.8) +} + +.swagger-ui .white-70 { + color: hsla(0,0%,100%,.7) +} + +.swagger-ui .white-60 { + color: hsla(0,0%,100%,.6) +} + +.swagger-ui .white-50 { + color: hsla(0,0%,100%,.5) +} + +.swagger-ui .white-40 { + color: hsla(0,0%,100%,.4) +} + +.swagger-ui .white-30 { + color: hsla(0,0%,100%,.3) +} + +.swagger-ui .white-20 { + color: hsla(0,0%,100%,.2) +} + +.swagger-ui .white-10 { + color: hsla(0,0%,100%,.1) +} + +.swagger-ui .black { + color: #000 +} + +.swagger-ui .near-black { + color: #111 +} + +.swagger-ui .dark-gray { + color: #333 +} + +.swagger-ui .mid-gray { + color: #555 +} + +.swagger-ui .gray { + color: #777 +} + +.swagger-ui .silver { + color: #999 +} + +.swagger-ui .light-silver { + color: #aaa +} + +.swagger-ui .moon-gray { + color: #ccc +} + +.swagger-ui .light-gray { + color: #eee +} + +.swagger-ui .near-white { + color: #f4f4f4 +} + +.swagger-ui .white { + color: #fff +} + +.swagger-ui .dark-red { + color: #e7040f +} + +.swagger-ui .red { + color: #ff4136 +} + +.swagger-ui .light-red { + color: #ff725c +} + +.swagger-ui .orange { + color: #ff6300 +} + +.swagger-ui .gold { + color: #ffb700 +} + +.swagger-ui .yellow { + color: gold +} + +.swagger-ui .light-yellow { + color: #fbf1a9 +} + +.swagger-ui .purple { + color: #5e2ca5 +} + +.swagger-ui .light-purple { + color: #a463f2 +} + +.swagger-ui .dark-pink { + color: #d5008f +} + +.swagger-ui .hot-pink { + color: #ff41b4 +} + +.swagger-ui .pink { + color: #ff80cc +} + +.swagger-ui .light-pink { + color: #ffa3d7 +} + +.swagger-ui .dark-green { + color: #137752 +} + +.swagger-ui .green { + color: #19a974 +} + +.swagger-ui .light-green { + color: #9eebcf +} + +.swagger-ui .navy { + color: #001b44 +} + +.swagger-ui .dark-blue { + color: #00449e +} + +.swagger-ui .blue { + color: #357edd +} + +.swagger-ui .light-blue { + color: #96ccff +} + +.swagger-ui .lightest-blue { + color: #cdecff +} + +.swagger-ui .washed-blue { + color: #f6fffe +} + +.swagger-ui .washed-green { + color: #e8fdf5 +} + +.swagger-ui .washed-yellow { + color: #fffceb +} + +.swagger-ui .washed-red { + color: #ffdfdf +} + +.swagger-ui .color-inherit { + color: inherit +} + +.swagger-ui .bg-black-90 { + background-color: rgba(0,0,0,.9) +} + +.swagger-ui .bg-black-80 { + background-color: rgba(0,0,0,.8) +} + +.swagger-ui .bg-black-70 { + background-color: rgba(0,0,0,.7) +} + +.swagger-ui .bg-black-60 { + background-color: rgba(0,0,0,.6) +} + +.swagger-ui .bg-black-50 { + background-color: rgba(0,0,0,.5) +} + +.swagger-ui .bg-black-40 { + background-color: rgba(0,0,0,.4) +} + +.swagger-ui .bg-black-30 { + background-color: rgba(0,0,0,.3) +} + +.swagger-ui .bg-black-20 { + background-color: rgba(0,0,0,.2) +} + +.swagger-ui .bg-black-10 { + background-color: rgba(0,0,0,.1) +} + +.swagger-ui .bg-black-05 { + background-color: rgba(0,0,0,.05) +} + +.swagger-ui .bg-white-90 { + background-color: hsla(0,0%,100%,.9) +} + +.swagger-ui .bg-white-80 { + background-color: hsla(0,0%,100%,.8) +} + +.swagger-ui .bg-white-70 { + background-color: hsla(0,0%,100%,.7) +} + +.swagger-ui .bg-white-60 { + background-color: hsla(0,0%,100%,.6) +} + +.swagger-ui .bg-white-50 { + background-color: hsla(0,0%,100%,.5) +} + +.swagger-ui .bg-white-40 { + background-color: hsla(0,0%,100%,.4) +} + +.swagger-ui .bg-white-30 { + background-color: hsla(0,0%,100%,.3) +} + +.swagger-ui .bg-white-20 { + background-color: hsla(0,0%,100%,.2) +} + +.swagger-ui .bg-white-10 { + background-color: hsla(0,0%,100%,.1) +} + +.swagger-ui .bg-black { + background-color: #000 +} + +.swagger-ui .bg-near-black { + background-color: #111 +} + +.swagger-ui .bg-dark-gray { + background-color: #333 +} + +.swagger-ui .bg-mid-gray { + background-color: #555 +} + +.swagger-ui .bg-gray { + background-color: #777 +} + +.swagger-ui .bg-silver { + background-color: #999 +} + +.swagger-ui .bg-light-silver { + background-color: #aaa +} + +.swagger-ui .bg-moon-gray { + background-color: #ccc +} + +.swagger-ui .bg-light-gray { + background-color: #eee +} + +.swagger-ui .bg-near-white { + background-color: #f4f4f4 +} + +.swagger-ui .bg-white { + background-color: #fff +} + +.swagger-ui .bg-transparent { + background-color: transparent +} + +.swagger-ui .bg-dark-red { + background-color: #e7040f +} + +.swagger-ui .bg-red { + background-color: #ff4136 +} + +.swagger-ui .bg-light-red { + background-color: #ff725c +} + +.swagger-ui .bg-orange { + background-color: #ff6300 +} + +.swagger-ui .bg-gold { + background-color: #ffb700 +} + +.swagger-ui .bg-yellow { + background-color: gold +} + +.swagger-ui .bg-light-yellow { + background-color: #fbf1a9 +} + +.swagger-ui .bg-purple { + background-color: #5e2ca5 +} + +.swagger-ui .bg-light-purple { + background-color: #a463f2 +} + +.swagger-ui .bg-dark-pink { + background-color: #d5008f +} + +.swagger-ui .bg-hot-pink { + background-color: #ff41b4 +} + +.swagger-ui .bg-pink { + background-color: #ff80cc +} + +.swagger-ui .bg-light-pink { + background-color: #ffa3d7 +} + +.swagger-ui .bg-dark-green { + background-color: #137752 +} + +.swagger-ui .bg-green { + background-color: #19a974 +} + +.swagger-ui .bg-light-green { + background-color: #9eebcf +} + +.swagger-ui .bg-navy { + background-color: #001b44 +} + +.swagger-ui .bg-dark-blue { + background-color: #00449e +} + +.swagger-ui .bg-blue { + background-color: #357edd +} + +.swagger-ui .bg-light-blue { + background-color: #96ccff +} + +.swagger-ui .bg-lightest-blue { + background-color: #cdecff +} + +.swagger-ui .bg-washed-blue { + background-color: #f6fffe +} + +.swagger-ui .bg-washed-green { + background-color: #e8fdf5 +} + +.swagger-ui .bg-washed-yellow { + background-color: #fffceb +} + +.swagger-ui .bg-washed-red { + background-color: #ffdfdf +} + +.swagger-ui .bg-inherit { + background-color: inherit +} + +.swagger-ui .hover-black:focus,.swagger-ui .hover-black:hover { + color: #000 +} + +.swagger-ui .hover-near-black:focus,.swagger-ui .hover-near-black:hover { + color: #111 +} + +.swagger-ui .hover-dark-gray:focus,.swagger-ui .hover-dark-gray:hover { + color: #333 +} + +.swagger-ui .hover-mid-gray:focus,.swagger-ui .hover-mid-gray:hover { + color: #555 +} + +.swagger-ui .hover-gray:focus,.swagger-ui .hover-gray:hover { + color: #777 +} + +.swagger-ui .hover-silver:focus,.swagger-ui .hover-silver:hover { + color: #999 +} + +.swagger-ui .hover-light-silver:focus,.swagger-ui .hover-light-silver:hover { + color: #aaa +} + +.swagger-ui .hover-moon-gray:focus,.swagger-ui .hover-moon-gray:hover { + color: #ccc +} + +.swagger-ui .hover-light-gray:focus,.swagger-ui .hover-light-gray:hover { + color: #eee +} + +.swagger-ui .hover-near-white:focus,.swagger-ui .hover-near-white:hover { + color: #f4f4f4 +} + +.swagger-ui .hover-white:focus,.swagger-ui .hover-white:hover { + color: #fff +} + +.swagger-ui .hover-black-90:focus,.swagger-ui .hover-black-90:hover { + color: rgba(0,0,0,.9) +} + +.swagger-ui .hover-black-80:focus,.swagger-ui .hover-black-80:hover { + color: rgba(0,0,0,.8) +} + +.swagger-ui .hover-black-70:focus,.swagger-ui .hover-black-70:hover { + color: rgba(0,0,0,.7) +} + +.swagger-ui .hover-black-60:focus,.swagger-ui .hover-black-60:hover { + color: rgba(0,0,0,.6) +} + +.swagger-ui .hover-black-50:focus,.swagger-ui .hover-black-50:hover { + color: rgba(0,0,0,.5) +} + +.swagger-ui .hover-black-40:focus,.swagger-ui .hover-black-40:hover { + color: rgba(0,0,0,.4) +} + +.swagger-ui .hover-black-30:focus,.swagger-ui .hover-black-30:hover { + color: rgba(0,0,0,.3) +} + +.swagger-ui .hover-black-20:focus,.swagger-ui .hover-black-20:hover { + color: rgba(0,0,0,.2) +} + +.swagger-ui .hover-black-10:focus,.swagger-ui .hover-black-10:hover { + color: rgba(0,0,0,.1) +} + +.swagger-ui .hover-white-90:focus,.swagger-ui .hover-white-90:hover { + color: hsla(0,0%,100%,.9) +} + +.swagger-ui .hover-white-80:focus,.swagger-ui .hover-white-80:hover { + color: hsla(0,0%,100%,.8) +} + +.swagger-ui .hover-white-70:focus,.swagger-ui .hover-white-70:hover { + color: hsla(0,0%,100%,.7) +} + +.swagger-ui .hover-white-60:focus,.swagger-ui .hover-white-60:hover { + color: hsla(0,0%,100%,.6) +} + +.swagger-ui .hover-white-50:focus,.swagger-ui .hover-white-50:hover { + color: hsla(0,0%,100%,.5) +} + +.swagger-ui .hover-white-40:focus,.swagger-ui .hover-white-40:hover { + color: hsla(0,0%,100%,.4) +} + +.swagger-ui .hover-white-30:focus,.swagger-ui .hover-white-30:hover { + color: hsla(0,0%,100%,.3) +} + +.swagger-ui .hover-white-20:focus,.swagger-ui .hover-white-20:hover { + color: hsla(0,0%,100%,.2) +} + +.swagger-ui .hover-white-10:focus,.swagger-ui .hover-white-10:hover { + color: hsla(0,0%,100%,.1) +} + +.swagger-ui .hover-inherit:focus,.swagger-ui .hover-inherit:hover { + color: inherit +} + +.swagger-ui .hover-bg-black:focus,.swagger-ui .hover-bg-black:hover { + background-color: #000 +} + +.swagger-ui .hover-bg-near-black:focus,.swagger-ui .hover-bg-near-black:hover { + background-color: #111 +} + +.swagger-ui .hover-bg-dark-gray:focus,.swagger-ui .hover-bg-dark-gray:hover { + background-color: #333 +} + +.swagger-ui .hover-bg-mid-gray:focus,.swagger-ui .hover-bg-mid-gray:hover { + background-color: #555 +} + +.swagger-ui .hover-bg-gray:focus,.swagger-ui .hover-bg-gray:hover { + background-color: #777 +} + +.swagger-ui .hover-bg-silver:focus,.swagger-ui .hover-bg-silver:hover { + background-color: #999 +} + +.swagger-ui .hover-bg-light-silver:focus,.swagger-ui .hover-bg-light-silver:hover { + background-color: #aaa +} + +.swagger-ui .hover-bg-moon-gray:focus,.swagger-ui .hover-bg-moon-gray:hover { + background-color: #ccc +} + +.swagger-ui .hover-bg-light-gray:focus,.swagger-ui .hover-bg-light-gray:hover { + background-color: #eee +} + +.swagger-ui .hover-bg-near-white:focus,.swagger-ui .hover-bg-near-white:hover { + background-color: #f4f4f4 +} + +.swagger-ui .hover-bg-white:focus,.swagger-ui .hover-bg-white:hover { + background-color: #fff +} + +.swagger-ui .hover-bg-transparent:focus,.swagger-ui .hover-bg-transparent:hover { + background-color: transparent +} + +.swagger-ui .hover-bg-black-90:focus,.swagger-ui .hover-bg-black-90:hover { + background-color: rgba(0,0,0,.9) +} + +.swagger-ui .hover-bg-black-80:focus,.swagger-ui .hover-bg-black-80:hover { + background-color: rgba(0,0,0,.8) +} + +.swagger-ui .hover-bg-black-70:focus,.swagger-ui .hover-bg-black-70:hover { + background-color: rgba(0,0,0,.7) +} + +.swagger-ui .hover-bg-black-60:focus,.swagger-ui .hover-bg-black-60:hover { + background-color: rgba(0,0,0,.6) +} + +.swagger-ui .hover-bg-black-50:focus,.swagger-ui .hover-bg-black-50:hover { + background-color: rgba(0,0,0,.5) +} + +.swagger-ui .hover-bg-black-40:focus,.swagger-ui .hover-bg-black-40:hover { + background-color: rgba(0,0,0,.4) +} + +.swagger-ui .hover-bg-black-30:focus,.swagger-ui .hover-bg-black-30:hover { + background-color: rgba(0,0,0,.3) +} + +.swagger-ui .hover-bg-black-20:focus,.swagger-ui .hover-bg-black-20:hover { + background-color: rgba(0,0,0,.2) +} + +.swagger-ui .hover-bg-black-10:focus,.swagger-ui .hover-bg-black-10:hover { + background-color: rgba(0,0,0,.1) +} + +.swagger-ui .hover-bg-white-90:focus,.swagger-ui .hover-bg-white-90:hover { + background-color: hsla(0,0%,100%,.9) +} + +.swagger-ui .hover-bg-white-80:focus,.swagger-ui .hover-bg-white-80:hover { + background-color: hsla(0,0%,100%,.8) +} + +.swagger-ui .hover-bg-white-70:focus,.swagger-ui .hover-bg-white-70:hover { + background-color: hsla(0,0%,100%,.7) +} + +.swagger-ui .hover-bg-white-60:focus,.swagger-ui .hover-bg-white-60:hover { + background-color: hsla(0,0%,100%,.6) +} + +.swagger-ui .hover-bg-white-50:focus,.swagger-ui .hover-bg-white-50:hover { + background-color: hsla(0,0%,100%,.5) +} + +.swagger-ui .hover-bg-white-40:focus,.swagger-ui .hover-bg-white-40:hover { + background-color: hsla(0,0%,100%,.4) +} + +.swagger-ui .hover-bg-white-30:focus,.swagger-ui .hover-bg-white-30:hover { + background-color: hsla(0,0%,100%,.3) +} + +.swagger-ui .hover-bg-white-20:focus,.swagger-ui .hover-bg-white-20:hover { + background-color: hsla(0,0%,100%,.2) +} + +.swagger-ui .hover-bg-white-10:focus,.swagger-ui .hover-bg-white-10:hover { + background-color: hsla(0,0%,100%,.1) +} + +.swagger-ui .hover-dark-red:focus,.swagger-ui .hover-dark-red:hover { + color: #e7040f +} + +.swagger-ui .hover-red:focus,.swagger-ui .hover-red:hover { + color: #ff4136 +} + +.swagger-ui .hover-light-red:focus,.swagger-ui .hover-light-red:hover { + color: #ff725c +} + +.swagger-ui .hover-orange:focus,.swagger-ui .hover-orange:hover { + color: #ff6300 +} + +.swagger-ui .hover-gold:focus,.swagger-ui .hover-gold:hover { + color: #ffb700 +} + +.swagger-ui .hover-yellow:focus,.swagger-ui .hover-yellow:hover { + color: gold +} + +.swagger-ui .hover-light-yellow:focus,.swagger-ui .hover-light-yellow:hover { + color: #fbf1a9 +} + +.swagger-ui .hover-purple:focus,.swagger-ui .hover-purple:hover { + color: #5e2ca5 +} + +.swagger-ui .hover-light-purple:focus,.swagger-ui .hover-light-purple:hover { + color: #a463f2 +} + +.swagger-ui .hover-dark-pink:focus,.swagger-ui .hover-dark-pink:hover { + color: #d5008f +} + +.swagger-ui .hover-hot-pink:focus,.swagger-ui .hover-hot-pink:hover { + color: #ff41b4 +} + +.swagger-ui .hover-pink:focus,.swagger-ui .hover-pink:hover { + color: #ff80cc +} + +.swagger-ui .hover-light-pink:focus,.swagger-ui .hover-light-pink:hover { + color: #ffa3d7 +} + +.swagger-ui .hover-dark-green:focus,.swagger-ui .hover-dark-green:hover { + color: #137752 +} + +.swagger-ui .hover-green:focus,.swagger-ui .hover-green:hover { + color: #19a974 +} + +.swagger-ui .hover-light-green:focus,.swagger-ui .hover-light-green:hover { + color: #9eebcf +} + +.swagger-ui .hover-navy:focus,.swagger-ui .hover-navy:hover { + color: #001b44 +} + +.swagger-ui .hover-dark-blue:focus,.swagger-ui .hover-dark-blue:hover { + color: #00449e +} + +.swagger-ui .hover-blue:focus,.swagger-ui .hover-blue:hover { + color: #357edd +} + +.swagger-ui .hover-light-blue:focus,.swagger-ui .hover-light-blue:hover { + color: #96ccff +} + +.swagger-ui .hover-lightest-blue:focus,.swagger-ui .hover-lightest-blue:hover { + color: #cdecff +} + +.swagger-ui .hover-washed-blue:focus,.swagger-ui .hover-washed-blue:hover { + color: #f6fffe +} + +.swagger-ui .hover-washed-green:focus,.swagger-ui .hover-washed-green:hover { + color: #e8fdf5 +} + +.swagger-ui .hover-washed-yellow:focus,.swagger-ui .hover-washed-yellow:hover { + color: #fffceb +} + +.swagger-ui .hover-washed-red:focus,.swagger-ui .hover-washed-red:hover { + color: #ffdfdf +} + +.swagger-ui .hover-bg-dark-red:focus,.swagger-ui .hover-bg-dark-red:hover { + background-color: #e7040f +} + +.swagger-ui .hover-bg-red:focus,.swagger-ui .hover-bg-red:hover { + background-color: #ff4136 +} + +.swagger-ui .hover-bg-light-red:focus,.swagger-ui .hover-bg-light-red:hover { + background-color: #ff725c +} + +.swagger-ui .hover-bg-orange:focus,.swagger-ui .hover-bg-orange:hover { + background-color: #ff6300 +} + +.swagger-ui .hover-bg-gold:focus,.swagger-ui .hover-bg-gold:hover { + background-color: #ffb700 +} + +.swagger-ui .hover-bg-yellow:focus,.swagger-ui .hover-bg-yellow:hover { + background-color: gold +} + +.swagger-ui .hover-bg-light-yellow:focus,.swagger-ui .hover-bg-light-yellow:hover { + background-color: #fbf1a9 +} + +.swagger-ui .hover-bg-purple:focus,.swagger-ui .hover-bg-purple:hover { + background-color: #5e2ca5 +} + +.swagger-ui .hover-bg-light-purple:focus,.swagger-ui .hover-bg-light-purple:hover { + background-color: #a463f2 +} + +.swagger-ui .hover-bg-dark-pink:focus,.swagger-ui .hover-bg-dark-pink:hover { + background-color: #d5008f +} + +.swagger-ui .hover-bg-hot-pink:focus,.swagger-ui .hover-bg-hot-pink:hover { + background-color: #ff41b4 +} + +.swagger-ui .hover-bg-pink:focus,.swagger-ui .hover-bg-pink:hover { + background-color: #ff80cc +} + +.swagger-ui .hover-bg-light-pink:focus,.swagger-ui .hover-bg-light-pink:hover { + background-color: #ffa3d7 +} + +.swagger-ui .hover-bg-dark-green:focus,.swagger-ui .hover-bg-dark-green:hover { + background-color: #137752 +} + +.swagger-ui .hover-bg-green:focus,.swagger-ui .hover-bg-green:hover { + background-color: #19a974 +} + +.swagger-ui .hover-bg-light-green:focus,.swagger-ui .hover-bg-light-green:hover { + background-color: #9eebcf +} + +.swagger-ui .hover-bg-navy:focus,.swagger-ui .hover-bg-navy:hover { + background-color: #001b44 +} + +.swagger-ui .hover-bg-dark-blue:focus,.swagger-ui .hover-bg-dark-blue:hover { + background-color: #00449e +} + +.swagger-ui .hover-bg-blue:focus,.swagger-ui .hover-bg-blue:hover { + background-color: #357edd +} + +.swagger-ui .hover-bg-light-blue:focus,.swagger-ui .hover-bg-light-blue:hover { + background-color: #96ccff +} + +.swagger-ui .hover-bg-lightest-blue:focus,.swagger-ui .hover-bg-lightest-blue:hover { + background-color: #cdecff +} + +.swagger-ui .hover-bg-washed-blue:focus,.swagger-ui .hover-bg-washed-blue:hover { + background-color: #f6fffe +} + +.swagger-ui .hover-bg-washed-green:focus,.swagger-ui .hover-bg-washed-green:hover { + background-color: #e8fdf5 +} + +.swagger-ui .hover-bg-washed-yellow:focus,.swagger-ui .hover-bg-washed-yellow:hover { + background-color: #fffceb +} + +.swagger-ui .hover-bg-washed-red:focus,.swagger-ui .hover-bg-washed-red:hover { + background-color: #ffdfdf +} + +.swagger-ui .hover-bg-inherit:focus,.swagger-ui .hover-bg-inherit:hover { + background-color: inherit +} + +.swagger-ui .pa0 { + padding: 0 +} + +.swagger-ui .pa1 { + padding: .25rem +} + +.swagger-ui .pa2 { + padding: .5rem +} + +.swagger-ui .pa3 { + padding: 1rem +} + +.swagger-ui .pa4 { + padding: 2rem +} + +.swagger-ui .pa5 { + padding: 4rem +} + +.swagger-ui .pa6 { + padding: 8rem +} + +.swagger-ui .pa7 { + padding: 16rem +} + +.swagger-ui .pl0 { + padding-left: 0 +} + +.swagger-ui .pl1 { + padding-left: .25rem +} + +.swagger-ui .pl2 { + padding-left: .5rem +} + +.swagger-ui .pl3 { + padding-left: 1rem +} + +.swagger-ui .pl4 { + padding-left: 2rem +} + +.swagger-ui .pl5 { + padding-left: 4rem +} + +.swagger-ui .pl6 { + padding-left: 8rem +} + +.swagger-ui .pl7 { + padding-left: 16rem +} + +.swagger-ui .pr0 { + padding-right: 0 +} + +.swagger-ui .pr1 { + padding-right: .25rem +} + +.swagger-ui .pr2 { + padding-right: .5rem +} + +.swagger-ui .pr3 { + padding-right: 1rem +} + +.swagger-ui .pr4 { + padding-right: 2rem +} + +.swagger-ui .pr5 { + padding-right: 4rem +} + +.swagger-ui .pr6 { + padding-right: 8rem +} + +.swagger-ui .pr7 { + padding-right: 16rem +} + +.swagger-ui .pb0 { + padding-bottom: 0 +} + +.swagger-ui .pb1 { + padding-bottom: .25rem +} + +.swagger-ui .pb2 { + padding-bottom: .5rem +} + +.swagger-ui .pb3 { + padding-bottom: 1rem +} + +.swagger-ui .pb4 { + padding-bottom: 2rem +} + +.swagger-ui .pb5 { + padding-bottom: 4rem +} + +.swagger-ui .pb6 { + padding-bottom: 8rem +} + +.swagger-ui .pb7 { + padding-bottom: 16rem +} + +.swagger-ui .pt0 { + padding-top: 0 +} + +.swagger-ui .pt1 { + padding-top: .25rem +} + +.swagger-ui .pt2 { + padding-top: .5rem +} + +.swagger-ui .pt3 { + padding-top: 1rem +} + +.swagger-ui .pt4 { + padding-top: 2rem +} + +.swagger-ui .pt5 { + padding-top: 4rem +} + +.swagger-ui .pt6 { + padding-top: 8rem +} + +.swagger-ui .pt7 { + padding-top: 16rem +} + +.swagger-ui .pv0 { + padding-bottom: 0; + padding-top: 0 +} + +.swagger-ui .pv1 { + padding-bottom: .25rem; + padding-top: .25rem +} + +.swagger-ui .pv2 { + padding-bottom: .5rem; + padding-top: .5rem +} + +.swagger-ui .pv3 { + padding-bottom: 1rem; + padding-top: 1rem +} + +.swagger-ui .pv4 { + padding-bottom: 2rem; + padding-top: 2rem +} + +.swagger-ui .pv5 { + padding-bottom: 4rem; + padding-top: 4rem +} + +.swagger-ui .pv6 { + padding-bottom: 8rem; + padding-top: 8rem +} + +.swagger-ui .pv7 { + padding-bottom: 16rem; + padding-top: 16rem +} + +.swagger-ui .ph0 { + padding-left: 0; + padding-right: 0 +} + +.swagger-ui .ph1 { + padding-left: .25rem; + padding-right: .25rem +} + +.swagger-ui .ph2 { + padding-left: .5rem; + padding-right: .5rem +} + +.swagger-ui .ph3 { + padding-left: 1rem; + padding-right: 1rem +} + +.swagger-ui .ph4 { + padding-left: 2rem; + padding-right: 2rem +} + +.swagger-ui .ph5 { + padding-left: 4rem; + padding-right: 4rem +} + +.swagger-ui .ph6 { + padding-left: 8rem; + padding-right: 8rem +} + +.swagger-ui .ph7 { + padding-left: 16rem; + padding-right: 16rem +} + +.swagger-ui .ma0 { + margin: 0 +} + +.swagger-ui .ma1 { + margin: .25rem +} + +.swagger-ui .ma2 { + margin: .5rem +} + +.swagger-ui .ma3 { + margin: 1rem +} + +.swagger-ui .ma4 { + margin: 2rem +} + +.swagger-ui .ma5 { + margin: 4rem +} + +.swagger-ui .ma6 { + margin: 8rem +} + +.swagger-ui .ma7 { + margin: 16rem +} + +.swagger-ui .ml0 { + margin-left: 0 +} + +.swagger-ui .ml1 { + margin-left: .25rem +} + +.swagger-ui .ml2 { + margin-left: .5rem +} + +.swagger-ui .ml3 { + margin-left: 1rem +} + +.swagger-ui .ml4 { + margin-left: 2rem +} + +.swagger-ui .ml5 { + margin-left: 4rem +} + +.swagger-ui .ml6 { + margin-left: 8rem +} + +.swagger-ui .ml7 { + margin-left: 16rem +} + +.swagger-ui .mr0 { + margin-right: 0 +} + +.swagger-ui .mr1 { + margin-right: .25rem +} + +.swagger-ui .mr2 { + margin-right: .5rem +} + +.swagger-ui .mr3 { + margin-right: 1rem +} + +.swagger-ui .mr4 { + margin-right: 2rem +} + +.swagger-ui .mr5 { + margin-right: 4rem +} + +.swagger-ui .mr6 { + margin-right: 8rem +} + +.swagger-ui .mr7 { + margin-right: 16rem +} + +.swagger-ui .mb0 { + margin-bottom: 0 +} + +.swagger-ui .mb1 { + margin-bottom: .25rem +} + +.swagger-ui .mb2 { + margin-bottom: .5rem +} + +.swagger-ui .mb3 { + margin-bottom: 1rem +} + +.swagger-ui .mb4 { + margin-bottom: 2rem +} + +.swagger-ui .mb5 { + margin-bottom: 4rem +} + +.swagger-ui .mb6 { + margin-bottom: 8rem +} + +.swagger-ui .mb7 { + margin-bottom: 16rem +} + +.swagger-ui .mt0 { + margin-top: 0 +} + +.swagger-ui .mt1 { + margin-top: .25rem +} + +.swagger-ui .mt2 { + margin-top: .5rem +} + +.swagger-ui .mt3 { + margin-top: 1rem +} + +.swagger-ui .mt4 { + margin-top: 2rem +} + +.swagger-ui .mt5 { + margin-top: 4rem +} + +.swagger-ui .mt6 { + margin-top: 8rem +} + +.swagger-ui .mt7 { + margin-top: 16rem +} + +.swagger-ui .mv0 { + margin-bottom: 0; + margin-top: 0 +} + +.swagger-ui .mv1 { + margin-bottom: .25rem; + margin-top: .25rem +} + +.swagger-ui .mv2 { + margin-bottom: .5rem; + margin-top: .5rem +} + +.swagger-ui .mv3 { + margin-bottom: 1rem; + margin-top: 1rem +} + +.swagger-ui .mv4 { + margin-bottom: 2rem; + margin-top: 2rem +} + +.swagger-ui .mv5 { + margin-bottom: 4rem; + margin-top: 4rem +} + +.swagger-ui .mv6 { + margin-bottom: 8rem; + margin-top: 8rem +} + +.swagger-ui .mv7 { + margin-bottom: 16rem; + margin-top: 16rem +} + +.swagger-ui .mh0 { + margin-left: 0; + margin-right: 0 +} + +.swagger-ui .mh1 { + margin-left: .25rem; + margin-right: .25rem +} + +.swagger-ui .mh2 { + margin-left: .5rem; + margin-right: .5rem +} + +.swagger-ui .mh3 { + margin-left: 1rem; + margin-right: 1rem +} + +.swagger-ui .mh4 { + margin-left: 2rem; + margin-right: 2rem +} + +.swagger-ui .mh5 { + margin-left: 4rem; + margin-right: 4rem +} + +.swagger-ui .mh6 { + margin-left: 8rem; + margin-right: 8rem +} + +.swagger-ui .mh7 { + margin-left: 16rem; + margin-right: 16rem +} + +@media screen and (min-width: 30em) { + .swagger-ui .pa0-ns { + padding:0 + } + + .swagger-ui .pa1-ns { + padding: .25rem + } + + .swagger-ui .pa2-ns { + padding: .5rem + } + + .swagger-ui .pa3-ns { + padding: 1rem + } + + .swagger-ui .pa4-ns { + padding: 2rem + } + + .swagger-ui .pa5-ns { + padding: 4rem + } + + .swagger-ui .pa6-ns { + padding: 8rem + } + + .swagger-ui .pa7-ns { + padding: 16rem + } + + .swagger-ui .pl0-ns { + padding-left: 0 + } + + .swagger-ui .pl1-ns { + padding-left: .25rem + } + + .swagger-ui .pl2-ns { + padding-left: .5rem + } + + .swagger-ui .pl3-ns { + padding-left: 1rem + } + + .swagger-ui .pl4-ns { + padding-left: 2rem + } + + .swagger-ui .pl5-ns { + padding-left: 4rem + } + + .swagger-ui .pl6-ns { + padding-left: 8rem + } + + .swagger-ui .pl7-ns { + padding-left: 16rem + } + + .swagger-ui .pr0-ns { + padding-right: 0 + } + + .swagger-ui .pr1-ns { + padding-right: .25rem + } + + .swagger-ui .pr2-ns { + padding-right: .5rem + } + + .swagger-ui .pr3-ns { + padding-right: 1rem + } + + .swagger-ui .pr4-ns { + padding-right: 2rem + } + + .swagger-ui .pr5-ns { + padding-right: 4rem + } + + .swagger-ui .pr6-ns { + padding-right: 8rem + } + + .swagger-ui .pr7-ns { + padding-right: 16rem + } + + .swagger-ui .pb0-ns { + padding-bottom: 0 + } + + .swagger-ui .pb1-ns { + padding-bottom: .25rem + } + + .swagger-ui .pb2-ns { + padding-bottom: .5rem + } + + .swagger-ui .pb3-ns { + padding-bottom: 1rem + } + + .swagger-ui .pb4-ns { + padding-bottom: 2rem + } + + .swagger-ui .pb5-ns { + padding-bottom: 4rem + } + + .swagger-ui .pb6-ns { + padding-bottom: 8rem + } + + .swagger-ui .pb7-ns { + padding-bottom: 16rem + } + + .swagger-ui .pt0-ns { + padding-top: 0 + } + + .swagger-ui .pt1-ns { + padding-top: .25rem + } + + .swagger-ui .pt2-ns { + padding-top: .5rem + } + + .swagger-ui .pt3-ns { + padding-top: 1rem + } + + .swagger-ui .pt4-ns { + padding-top: 2rem + } + + .swagger-ui .pt5-ns { + padding-top: 4rem + } + + .swagger-ui .pt6-ns { + padding-top: 8rem + } + + .swagger-ui .pt7-ns { + padding-top: 16rem + } + + .swagger-ui .pv0-ns { + padding-bottom: 0; + padding-top: 0 + } + + .swagger-ui .pv1-ns { + padding-bottom: .25rem; + padding-top: .25rem + } + + .swagger-ui .pv2-ns { + padding-bottom: .5rem; + padding-top: .5rem + } + + .swagger-ui .pv3-ns { + padding-bottom: 1rem; + padding-top: 1rem + } + + .swagger-ui .pv4-ns { + padding-bottom: 2rem; + padding-top: 2rem + } + + .swagger-ui .pv5-ns { + padding-bottom: 4rem; + padding-top: 4rem + } + + .swagger-ui .pv6-ns { + padding-bottom: 8rem; + padding-top: 8rem + } + + .swagger-ui .pv7-ns { + padding-bottom: 16rem; + padding-top: 16rem + } + + .swagger-ui .ph0-ns { + padding-left: 0; + padding-right: 0 + } + + .swagger-ui .ph1-ns { + padding-left: .25rem; + padding-right: .25rem + } + + .swagger-ui .ph2-ns { + padding-left: .5rem; + padding-right: .5rem + } + + .swagger-ui .ph3-ns { + padding-left: 1rem; + padding-right: 1rem + } + + .swagger-ui .ph4-ns { + padding-left: 2rem; + padding-right: 2rem + } + + .swagger-ui .ph5-ns { + padding-left: 4rem; + padding-right: 4rem + } + + .swagger-ui .ph6-ns { + padding-left: 8rem; + padding-right: 8rem + } + + .swagger-ui .ph7-ns { + padding-left: 16rem; + padding-right: 16rem + } + + .swagger-ui .ma0-ns { + margin: 0 + } + + .swagger-ui .ma1-ns { + margin: .25rem + } + + .swagger-ui .ma2-ns { + margin: .5rem + } + + .swagger-ui .ma3-ns { + margin: 1rem + } + + .swagger-ui .ma4-ns { + margin: 2rem + } + + .swagger-ui .ma5-ns { + margin: 4rem + } + + .swagger-ui .ma6-ns { + margin: 8rem + } + + .swagger-ui .ma7-ns { + margin: 16rem + } + + .swagger-ui .ml0-ns { + margin-left: 0 + } + + .swagger-ui .ml1-ns { + margin-left: .25rem + } + + .swagger-ui .ml2-ns { + margin-left: .5rem + } + + .swagger-ui .ml3-ns { + margin-left: 1rem + } + + .swagger-ui .ml4-ns { + margin-left: 2rem + } + + .swagger-ui .ml5-ns { + margin-left: 4rem + } + + .swagger-ui .ml6-ns { + margin-left: 8rem + } + + .swagger-ui .ml7-ns { + margin-left: 16rem + } + + .swagger-ui .mr0-ns { + margin-right: 0 + } + + .swagger-ui .mr1-ns { + margin-right: .25rem + } + + .swagger-ui .mr2-ns { + margin-right: .5rem + } + + .swagger-ui .mr3-ns { + margin-right: 1rem + } + + .swagger-ui .mr4-ns { + margin-right: 2rem + } + + .swagger-ui .mr5-ns { + margin-right: 4rem + } + + .swagger-ui .mr6-ns { + margin-right: 8rem + } + + .swagger-ui .mr7-ns { + margin-right: 16rem + } + + .swagger-ui .mb0-ns { + margin-bottom: 0 + } + + .swagger-ui .mb1-ns { + margin-bottom: .25rem + } + + .swagger-ui .mb2-ns { + margin-bottom: .5rem + } + + .swagger-ui .mb3-ns { + margin-bottom: 1rem + } + + .swagger-ui .mb4-ns { + margin-bottom: 2rem + } + + .swagger-ui .mb5-ns { + margin-bottom: 4rem + } + + .swagger-ui .mb6-ns { + margin-bottom: 8rem + } + + .swagger-ui .mb7-ns { + margin-bottom: 16rem + } + + .swagger-ui .mt0-ns { + margin-top: 0 + } + + .swagger-ui .mt1-ns { + margin-top: .25rem + } + + .swagger-ui .mt2-ns { + margin-top: .5rem + } + + .swagger-ui .mt3-ns { + margin-top: 1rem + } + + .swagger-ui .mt4-ns { + margin-top: 2rem + } + + .swagger-ui .mt5-ns { + margin-top: 4rem + } + + .swagger-ui .mt6-ns { + margin-top: 8rem + } + + .swagger-ui .mt7-ns { + margin-top: 16rem + } + + .swagger-ui .mv0-ns { + margin-bottom: 0; + margin-top: 0 + } + + .swagger-ui .mv1-ns { + margin-bottom: .25rem; + margin-top: .25rem + } + + .swagger-ui .mv2-ns { + margin-bottom: .5rem; + margin-top: .5rem + } + + .swagger-ui .mv3-ns { + margin-bottom: 1rem; + margin-top: 1rem + } + + .swagger-ui .mv4-ns { + margin-bottom: 2rem; + margin-top: 2rem + } + + .swagger-ui .mv5-ns { + margin-bottom: 4rem; + margin-top: 4rem + } + + .swagger-ui .mv6-ns { + margin-bottom: 8rem; + margin-top: 8rem + } + + .swagger-ui .mv7-ns { + margin-bottom: 16rem; + margin-top: 16rem + } + + .swagger-ui .mh0-ns { + margin-left: 0; + margin-right: 0 + } + + .swagger-ui .mh1-ns { + margin-left: .25rem; + margin-right: .25rem + } + + .swagger-ui .mh2-ns { + margin-left: .5rem; + margin-right: .5rem + } + + .swagger-ui .mh3-ns { + margin-left: 1rem; + margin-right: 1rem + } + + .swagger-ui .mh4-ns { + margin-left: 2rem; + margin-right: 2rem + } + + .swagger-ui .mh5-ns { + margin-left: 4rem; + margin-right: 4rem + } + + .swagger-ui .mh6-ns { + margin-left: 8rem; + margin-right: 8rem + } + + .swagger-ui .mh7-ns { + margin-left: 16rem; + margin-right: 16rem + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .pa0-m { + padding:0 + } + + .swagger-ui .pa1-m { + padding: .25rem + } + + .swagger-ui .pa2-m { + padding: .5rem + } + + .swagger-ui .pa3-m { + padding: 1rem + } + + .swagger-ui .pa4-m { + padding: 2rem + } + + .swagger-ui .pa5-m { + padding: 4rem + } + + .swagger-ui .pa6-m { + padding: 8rem + } + + .swagger-ui .pa7-m { + padding: 16rem + } + + .swagger-ui .pl0-m { + padding-left: 0 + } + + .swagger-ui .pl1-m { + padding-left: .25rem + } + + .swagger-ui .pl2-m { + padding-left: .5rem + } + + .swagger-ui .pl3-m { + padding-left: 1rem + } + + .swagger-ui .pl4-m { + padding-left: 2rem + } + + .swagger-ui .pl5-m { + padding-left: 4rem + } + + .swagger-ui .pl6-m { + padding-left: 8rem + } + + .swagger-ui .pl7-m { + padding-left: 16rem + } + + .swagger-ui .pr0-m { + padding-right: 0 + } + + .swagger-ui .pr1-m { + padding-right: .25rem + } + + .swagger-ui .pr2-m { + padding-right: .5rem + } + + .swagger-ui .pr3-m { + padding-right: 1rem + } + + .swagger-ui .pr4-m { + padding-right: 2rem + } + + .swagger-ui .pr5-m { + padding-right: 4rem + } + + .swagger-ui .pr6-m { + padding-right: 8rem + } + + .swagger-ui .pr7-m { + padding-right: 16rem + } + + .swagger-ui .pb0-m { + padding-bottom: 0 + } + + .swagger-ui .pb1-m { + padding-bottom: .25rem + } + + .swagger-ui .pb2-m { + padding-bottom: .5rem + } + + .swagger-ui .pb3-m { + padding-bottom: 1rem + } + + .swagger-ui .pb4-m { + padding-bottom: 2rem + } + + .swagger-ui .pb5-m { + padding-bottom: 4rem + } + + .swagger-ui .pb6-m { + padding-bottom: 8rem + } + + .swagger-ui .pb7-m { + padding-bottom: 16rem + } + + .swagger-ui .pt0-m { + padding-top: 0 + } + + .swagger-ui .pt1-m { + padding-top: .25rem + } + + .swagger-ui .pt2-m { + padding-top: .5rem + } + + .swagger-ui .pt3-m { + padding-top: 1rem + } + + .swagger-ui .pt4-m { + padding-top: 2rem + } + + .swagger-ui .pt5-m { + padding-top: 4rem + } + + .swagger-ui .pt6-m { + padding-top: 8rem + } + + .swagger-ui .pt7-m { + padding-top: 16rem + } + + .swagger-ui .pv0-m { + padding-bottom: 0; + padding-top: 0 + } + + .swagger-ui .pv1-m { + padding-bottom: .25rem; + padding-top: .25rem + } + + .swagger-ui .pv2-m { + padding-bottom: .5rem; + padding-top: .5rem + } + + .swagger-ui .pv3-m { + padding-bottom: 1rem; + padding-top: 1rem + } + + .swagger-ui .pv4-m { + padding-bottom: 2rem; + padding-top: 2rem + } + + .swagger-ui .pv5-m { + padding-bottom: 4rem; + padding-top: 4rem + } + + .swagger-ui .pv6-m { + padding-bottom: 8rem; + padding-top: 8rem + } + + .swagger-ui .pv7-m { + padding-bottom: 16rem; + padding-top: 16rem + } + + .swagger-ui .ph0-m { + padding-left: 0; + padding-right: 0 + } + + .swagger-ui .ph1-m { + padding-left: .25rem; + padding-right: .25rem + } + + .swagger-ui .ph2-m { + padding-left: .5rem; + padding-right: .5rem + } + + .swagger-ui .ph3-m { + padding-left: 1rem; + padding-right: 1rem + } + + .swagger-ui .ph4-m { + padding-left: 2rem; + padding-right: 2rem + } + + .swagger-ui .ph5-m { + padding-left: 4rem; + padding-right: 4rem + } + + .swagger-ui .ph6-m { + padding-left: 8rem; + padding-right: 8rem + } + + .swagger-ui .ph7-m { + padding-left: 16rem; + padding-right: 16rem + } + + .swagger-ui .ma0-m { + margin: 0 + } + + .swagger-ui .ma1-m { + margin: .25rem + } + + .swagger-ui .ma2-m { + margin: .5rem + } + + .swagger-ui .ma3-m { + margin: 1rem + } + + .swagger-ui .ma4-m { + margin: 2rem + } + + .swagger-ui .ma5-m { + margin: 4rem + } + + .swagger-ui .ma6-m { + margin: 8rem + } + + .swagger-ui .ma7-m { + margin: 16rem + } + + .swagger-ui .ml0-m { + margin-left: 0 + } + + .swagger-ui .ml1-m { + margin-left: .25rem + } + + .swagger-ui .ml2-m { + margin-left: .5rem + } + + .swagger-ui .ml3-m { + margin-left: 1rem + } + + .swagger-ui .ml4-m { + margin-left: 2rem + } + + .swagger-ui .ml5-m { + margin-left: 4rem + } + + .swagger-ui .ml6-m { + margin-left: 8rem + } + + .swagger-ui .ml7-m { + margin-left: 16rem + } + + .swagger-ui .mr0-m { + margin-right: 0 + } + + .swagger-ui .mr1-m { + margin-right: .25rem + } + + .swagger-ui .mr2-m { + margin-right: .5rem + } + + .swagger-ui .mr3-m { + margin-right: 1rem + } + + .swagger-ui .mr4-m { + margin-right: 2rem + } + + .swagger-ui .mr5-m { + margin-right: 4rem + } + + .swagger-ui .mr6-m { + margin-right: 8rem + } + + .swagger-ui .mr7-m { + margin-right: 16rem + } + + .swagger-ui .mb0-m { + margin-bottom: 0 + } + + .swagger-ui .mb1-m { + margin-bottom: .25rem + } + + .swagger-ui .mb2-m { + margin-bottom: .5rem + } + + .swagger-ui .mb3-m { + margin-bottom: 1rem + } + + .swagger-ui .mb4-m { + margin-bottom: 2rem + } + + .swagger-ui .mb5-m { + margin-bottom: 4rem + } + + .swagger-ui .mb6-m { + margin-bottom: 8rem + } + + .swagger-ui .mb7-m { + margin-bottom: 16rem + } + + .swagger-ui .mt0-m { + margin-top: 0 + } + + .swagger-ui .mt1-m { + margin-top: .25rem + } + + .swagger-ui .mt2-m { + margin-top: .5rem + } + + .swagger-ui .mt3-m { + margin-top: 1rem + } + + .swagger-ui .mt4-m { + margin-top: 2rem + } + + .swagger-ui .mt5-m { + margin-top: 4rem + } + + .swagger-ui .mt6-m { + margin-top: 8rem + } + + .swagger-ui .mt7-m { + margin-top: 16rem + } + + .swagger-ui .mv0-m { + margin-bottom: 0; + margin-top: 0 + } + + .swagger-ui .mv1-m { + margin-bottom: .25rem; + margin-top: .25rem + } + + .swagger-ui .mv2-m { + margin-bottom: .5rem; + margin-top: .5rem + } + + .swagger-ui .mv3-m { + margin-bottom: 1rem; + margin-top: 1rem + } + + .swagger-ui .mv4-m { + margin-bottom: 2rem; + margin-top: 2rem + } + + .swagger-ui .mv5-m { + margin-bottom: 4rem; + margin-top: 4rem + } + + .swagger-ui .mv6-m { + margin-bottom: 8rem; + margin-top: 8rem + } + + .swagger-ui .mv7-m { + margin-bottom: 16rem; + margin-top: 16rem + } + + .swagger-ui .mh0-m { + margin-left: 0; + margin-right: 0 + } + + .swagger-ui .mh1-m { + margin-left: .25rem; + margin-right: .25rem + } + + .swagger-ui .mh2-m { + margin-left: .5rem; + margin-right: .5rem + } + + .swagger-ui .mh3-m { + margin-left: 1rem; + margin-right: 1rem + } + + .swagger-ui .mh4-m { + margin-left: 2rem; + margin-right: 2rem + } + + .swagger-ui .mh5-m { + margin-left: 4rem; + margin-right: 4rem + } + + .swagger-ui .mh6-m { + margin-left: 8rem; + margin-right: 8rem + } + + .swagger-ui .mh7-m { + margin-left: 16rem; + margin-right: 16rem + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .pa0-l { + padding:0 + } + + .swagger-ui .pa1-l { + padding: .25rem + } + + .swagger-ui .pa2-l { + padding: .5rem + } + + .swagger-ui .pa3-l { + padding: 1rem + } + + .swagger-ui .pa4-l { + padding: 2rem + } + + .swagger-ui .pa5-l { + padding: 4rem + } + + .swagger-ui .pa6-l { + padding: 8rem + } + + .swagger-ui .pa7-l { + padding: 16rem + } + + .swagger-ui .pl0-l { + padding-left: 0 + } + + .swagger-ui .pl1-l { + padding-left: .25rem + } + + .swagger-ui .pl2-l { + padding-left: .5rem + } + + .swagger-ui .pl3-l { + padding-left: 1rem + } + + .swagger-ui .pl4-l { + padding-left: 2rem + } + + .swagger-ui .pl5-l { + padding-left: 4rem + } + + .swagger-ui .pl6-l { + padding-left: 8rem + } + + .swagger-ui .pl7-l { + padding-left: 16rem + } + + .swagger-ui .pr0-l { + padding-right: 0 + } + + .swagger-ui .pr1-l { + padding-right: .25rem + } + + .swagger-ui .pr2-l { + padding-right: .5rem + } + + .swagger-ui .pr3-l { + padding-right: 1rem + } + + .swagger-ui .pr4-l { + padding-right: 2rem + } + + .swagger-ui .pr5-l { + padding-right: 4rem + } + + .swagger-ui .pr6-l { + padding-right: 8rem + } + + .swagger-ui .pr7-l { + padding-right: 16rem + } + + .swagger-ui .pb0-l { + padding-bottom: 0 + } + + .swagger-ui .pb1-l { + padding-bottom: .25rem + } + + .swagger-ui .pb2-l { + padding-bottom: .5rem + } + + .swagger-ui .pb3-l { + padding-bottom: 1rem + } + + .swagger-ui .pb4-l { + padding-bottom: 2rem + } + + .swagger-ui .pb5-l { + padding-bottom: 4rem + } + + .swagger-ui .pb6-l { + padding-bottom: 8rem + } + + .swagger-ui .pb7-l { + padding-bottom: 16rem + } + + .swagger-ui .pt0-l { + padding-top: 0 + } + + .swagger-ui .pt1-l { + padding-top: .25rem + } + + .swagger-ui .pt2-l { + padding-top: .5rem + } + + .swagger-ui .pt3-l { + padding-top: 1rem + } + + .swagger-ui .pt4-l { + padding-top: 2rem + } + + .swagger-ui .pt5-l { + padding-top: 4rem + } + + .swagger-ui .pt6-l { + padding-top: 8rem + } + + .swagger-ui .pt7-l { + padding-top: 16rem + } + + .swagger-ui .pv0-l { + padding-bottom: 0; + padding-top: 0 + } + + .swagger-ui .pv1-l { + padding-bottom: .25rem; + padding-top: .25rem + } + + .swagger-ui .pv2-l { + padding-bottom: .5rem; + padding-top: .5rem + } + + .swagger-ui .pv3-l { + padding-bottom: 1rem; + padding-top: 1rem + } + + .swagger-ui .pv4-l { + padding-bottom: 2rem; + padding-top: 2rem + } + + .swagger-ui .pv5-l { + padding-bottom: 4rem; + padding-top: 4rem + } + + .swagger-ui .pv6-l { + padding-bottom: 8rem; + padding-top: 8rem + } + + .swagger-ui .pv7-l { + padding-bottom: 16rem; + padding-top: 16rem + } + + .swagger-ui .ph0-l { + padding-left: 0; + padding-right: 0 + } + + .swagger-ui .ph1-l { + padding-left: .25rem; + padding-right: .25rem + } + + .swagger-ui .ph2-l { + padding-left: .5rem; + padding-right: .5rem + } + + .swagger-ui .ph3-l { + padding-left: 1rem; + padding-right: 1rem + } + + .swagger-ui .ph4-l { + padding-left: 2rem; + padding-right: 2rem + } + + .swagger-ui .ph5-l { + padding-left: 4rem; + padding-right: 4rem + } + + .swagger-ui .ph6-l { + padding-left: 8rem; + padding-right: 8rem + } + + .swagger-ui .ph7-l { + padding-left: 16rem; + padding-right: 16rem + } + + .swagger-ui .ma0-l { + margin: 0 + } + + .swagger-ui .ma1-l { + margin: .25rem + } + + .swagger-ui .ma2-l { + margin: .5rem + } + + .swagger-ui .ma3-l { + margin: 1rem + } + + .swagger-ui .ma4-l { + margin: 2rem + } + + .swagger-ui .ma5-l { + margin: 4rem + } + + .swagger-ui .ma6-l { + margin: 8rem + } + + .swagger-ui .ma7-l { + margin: 16rem + } + + .swagger-ui .ml0-l { + margin-left: 0 + } + + .swagger-ui .ml1-l { + margin-left: .25rem + } + + .swagger-ui .ml2-l { + margin-left: .5rem + } + + .swagger-ui .ml3-l { + margin-left: 1rem + } + + .swagger-ui .ml4-l { + margin-left: 2rem + } + + .swagger-ui .ml5-l { + margin-left: 4rem + } + + .swagger-ui .ml6-l { + margin-left: 8rem + } + + .swagger-ui .ml7-l { + margin-left: 16rem + } + + .swagger-ui .mr0-l { + margin-right: 0 + } + + .swagger-ui .mr1-l { + margin-right: .25rem + } + + .swagger-ui .mr2-l { + margin-right: .5rem + } + + .swagger-ui .mr3-l { + margin-right: 1rem + } + + .swagger-ui .mr4-l { + margin-right: 2rem + } + + .swagger-ui .mr5-l { + margin-right: 4rem + } + + .swagger-ui .mr6-l { + margin-right: 8rem + } + + .swagger-ui .mr7-l { + margin-right: 16rem + } + + .swagger-ui .mb0-l { + margin-bottom: 0 + } + + .swagger-ui .mb1-l { + margin-bottom: .25rem + } + + .swagger-ui .mb2-l { + margin-bottom: .5rem + } + + .swagger-ui .mb3-l { + margin-bottom: 1rem + } + + .swagger-ui .mb4-l { + margin-bottom: 2rem + } + + .swagger-ui .mb5-l { + margin-bottom: 4rem + } + + .swagger-ui .mb6-l { + margin-bottom: 8rem + } + + .swagger-ui .mb7-l { + margin-bottom: 16rem + } + + .swagger-ui .mt0-l { + margin-top: 0 + } + + .swagger-ui .mt1-l { + margin-top: .25rem + } + + .swagger-ui .mt2-l { + margin-top: .5rem + } + + .swagger-ui .mt3-l { + margin-top: 1rem + } + + .swagger-ui .mt4-l { + margin-top: 2rem + } + + .swagger-ui .mt5-l { + margin-top: 4rem + } + + .swagger-ui .mt6-l { + margin-top: 8rem + } + + .swagger-ui .mt7-l { + margin-top: 16rem + } + + .swagger-ui .mv0-l { + margin-bottom: 0; + margin-top: 0 + } + + .swagger-ui .mv1-l { + margin-bottom: .25rem; + margin-top: .25rem + } + + .swagger-ui .mv2-l { + margin-bottom: .5rem; + margin-top: .5rem + } + + .swagger-ui .mv3-l { + margin-bottom: 1rem; + margin-top: 1rem + } + + .swagger-ui .mv4-l { + margin-bottom: 2rem; + margin-top: 2rem + } + + .swagger-ui .mv5-l { + margin-bottom: 4rem; + margin-top: 4rem + } + + .swagger-ui .mv6-l { + margin-bottom: 8rem; + margin-top: 8rem + } + + .swagger-ui .mv7-l { + margin-bottom: 16rem; + margin-top: 16rem + } + + .swagger-ui .mh0-l { + margin-left: 0; + margin-right: 0 + } + + .swagger-ui .mh1-l { + margin-left: .25rem; + margin-right: .25rem + } + + .swagger-ui .mh2-l { + margin-left: .5rem; + margin-right: .5rem + } + + .swagger-ui .mh3-l { + margin-left: 1rem; + margin-right: 1rem + } + + .swagger-ui .mh4-l { + margin-left: 2rem; + margin-right: 2rem + } + + .swagger-ui .mh5-l { + margin-left: 4rem; + margin-right: 4rem + } + + .swagger-ui .mh6-l { + margin-left: 8rem; + margin-right: 8rem + } + + .swagger-ui .mh7-l { + margin-left: 16rem; + margin-right: 16rem + } +} + +.swagger-ui .na1 { + margin: -.25rem +} + +.swagger-ui .na2 { + margin: -.5rem +} + +.swagger-ui .na3 { + margin: -1rem +} + +.swagger-ui .na4 { + margin: -2rem +} + +.swagger-ui .na5 { + margin: -4rem +} + +.swagger-ui .na6 { + margin: -8rem +} + +.swagger-ui .na7 { + margin: -16rem +} + +.swagger-ui .nl1 { + margin-left: -.25rem +} + +.swagger-ui .nl2 { + margin-left: -.5rem +} + +.swagger-ui .nl3 { + margin-left: -1rem +} + +.swagger-ui .nl4 { + margin-left: -2rem +} + +.swagger-ui .nl5 { + margin-left: -4rem +} + +.swagger-ui .nl6 { + margin-left: -8rem +} + +.swagger-ui .nl7 { + margin-left: -16rem +} + +.swagger-ui .nr1 { + margin-right: -.25rem +} + +.swagger-ui .nr2 { + margin-right: -.5rem +} + +.swagger-ui .nr3 { + margin-right: -1rem +} + +.swagger-ui .nr4 { + margin-right: -2rem +} + +.swagger-ui .nr5 { + margin-right: -4rem +} + +.swagger-ui .nr6 { + margin-right: -8rem +} + +.swagger-ui .nr7 { + margin-right: -16rem +} + +.swagger-ui .nb1 { + margin-bottom: -.25rem +} + +.swagger-ui .nb2 { + margin-bottom: -.5rem +} + +.swagger-ui .nb3 { + margin-bottom: -1rem +} + +.swagger-ui .nb4 { + margin-bottom: -2rem +} + +.swagger-ui .nb5 { + margin-bottom: -4rem +} + +.swagger-ui .nb6 { + margin-bottom: -8rem +} + +.swagger-ui .nb7 { + margin-bottom: -16rem +} + +.swagger-ui .nt1 { + margin-top: -.25rem +} + +.swagger-ui .nt2 { + margin-top: -.5rem +} + +.swagger-ui .nt3 { + margin-top: -1rem +} + +.swagger-ui .nt4 { + margin-top: -2rem +} + +.swagger-ui .nt5 { + margin-top: -4rem +} + +.swagger-ui .nt6 { + margin-top: -8rem +} + +.swagger-ui .nt7 { + margin-top: -16rem +} + +@media screen and (min-width: 30em) { + .swagger-ui .na1-ns { + margin:-.25rem + } + + .swagger-ui .na2-ns { + margin: -.5rem + } + + .swagger-ui .na3-ns { + margin: -1rem + } + + .swagger-ui .na4-ns { + margin: -2rem + } + + .swagger-ui .na5-ns { + margin: -4rem + } + + .swagger-ui .na6-ns { + margin: -8rem + } + + .swagger-ui .na7-ns { + margin: -16rem + } + + .swagger-ui .nl1-ns { + margin-left: -.25rem + } + + .swagger-ui .nl2-ns { + margin-left: -.5rem + } + + .swagger-ui .nl3-ns { + margin-left: -1rem + } + + .swagger-ui .nl4-ns { + margin-left: -2rem + } + + .swagger-ui .nl5-ns { + margin-left: -4rem + } + + .swagger-ui .nl6-ns { + margin-left: -8rem + } + + .swagger-ui .nl7-ns { + margin-left: -16rem + } + + .swagger-ui .nr1-ns { + margin-right: -.25rem + } + + .swagger-ui .nr2-ns { + margin-right: -.5rem + } + + .swagger-ui .nr3-ns { + margin-right: -1rem + } + + .swagger-ui .nr4-ns { + margin-right: -2rem + } + + .swagger-ui .nr5-ns { + margin-right: -4rem + } + + .swagger-ui .nr6-ns { + margin-right: -8rem + } + + .swagger-ui .nr7-ns { + margin-right: -16rem + } + + .swagger-ui .nb1-ns { + margin-bottom: -.25rem + } + + .swagger-ui .nb2-ns { + margin-bottom: -.5rem + } + + .swagger-ui .nb3-ns { + margin-bottom: -1rem + } + + .swagger-ui .nb4-ns { + margin-bottom: -2rem + } + + .swagger-ui .nb5-ns { + margin-bottom: -4rem + } + + .swagger-ui .nb6-ns { + margin-bottom: -8rem + } + + .swagger-ui .nb7-ns { + margin-bottom: -16rem + } + + .swagger-ui .nt1-ns { + margin-top: -.25rem + } + + .swagger-ui .nt2-ns { + margin-top: -.5rem + } + + .swagger-ui .nt3-ns { + margin-top: -1rem + } + + .swagger-ui .nt4-ns { + margin-top: -2rem + } + + .swagger-ui .nt5-ns { + margin-top: -4rem + } + + .swagger-ui .nt6-ns { + margin-top: -8rem + } + + .swagger-ui .nt7-ns { + margin-top: -16rem + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .na1-m { + margin:-.25rem + } + + .swagger-ui .na2-m { + margin: -.5rem + } + + .swagger-ui .na3-m { + margin: -1rem + } + + .swagger-ui .na4-m { + margin: -2rem + } + + .swagger-ui .na5-m { + margin: -4rem + } + + .swagger-ui .na6-m { + margin: -8rem + } + + .swagger-ui .na7-m { + margin: -16rem + } + + .swagger-ui .nl1-m { + margin-left: -.25rem + } + + .swagger-ui .nl2-m { + margin-left: -.5rem + } + + .swagger-ui .nl3-m { + margin-left: -1rem + } + + .swagger-ui .nl4-m { + margin-left: -2rem + } + + .swagger-ui .nl5-m { + margin-left: -4rem + } + + .swagger-ui .nl6-m { + margin-left: -8rem + } + + .swagger-ui .nl7-m { + margin-left: -16rem + } + + .swagger-ui .nr1-m { + margin-right: -.25rem + } + + .swagger-ui .nr2-m { + margin-right: -.5rem + } + + .swagger-ui .nr3-m { + margin-right: -1rem + } + + .swagger-ui .nr4-m { + margin-right: -2rem + } + + .swagger-ui .nr5-m { + margin-right: -4rem + } + + .swagger-ui .nr6-m { + margin-right: -8rem + } + + .swagger-ui .nr7-m { + margin-right: -16rem + } + + .swagger-ui .nb1-m { + margin-bottom: -.25rem + } + + .swagger-ui .nb2-m { + margin-bottom: -.5rem + } + + .swagger-ui .nb3-m { + margin-bottom: -1rem + } + + .swagger-ui .nb4-m { + margin-bottom: -2rem + } + + .swagger-ui .nb5-m { + margin-bottom: -4rem + } + + .swagger-ui .nb6-m { + margin-bottom: -8rem + } + + .swagger-ui .nb7-m { + margin-bottom: -16rem + } + + .swagger-ui .nt1-m { + margin-top: -.25rem + } + + .swagger-ui .nt2-m { + margin-top: -.5rem + } + + .swagger-ui .nt3-m { + margin-top: -1rem + } + + .swagger-ui .nt4-m { + margin-top: -2rem + } + + .swagger-ui .nt5-m { + margin-top: -4rem + } + + .swagger-ui .nt6-m { + margin-top: -8rem + } + + .swagger-ui .nt7-m { + margin-top: -16rem + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .na1-l { + margin:-.25rem + } + + .swagger-ui .na2-l { + margin: -.5rem + } + + .swagger-ui .na3-l { + margin: -1rem + } + + .swagger-ui .na4-l { + margin: -2rem + } + + .swagger-ui .na5-l { + margin: -4rem + } + + .swagger-ui .na6-l { + margin: -8rem + } + + .swagger-ui .na7-l { + margin: -16rem + } + + .swagger-ui .nl1-l { + margin-left: -.25rem + } + + .swagger-ui .nl2-l { + margin-left: -.5rem + } + + .swagger-ui .nl3-l { + margin-left: -1rem + } + + .swagger-ui .nl4-l { + margin-left: -2rem + } + + .swagger-ui .nl5-l { + margin-left: -4rem + } + + .swagger-ui .nl6-l { + margin-left: -8rem + } + + .swagger-ui .nl7-l { + margin-left: -16rem + } + + .swagger-ui .nr1-l { + margin-right: -.25rem + } + + .swagger-ui .nr2-l { + margin-right: -.5rem + } + + .swagger-ui .nr3-l { + margin-right: -1rem + } + + .swagger-ui .nr4-l { + margin-right: -2rem + } + + .swagger-ui .nr5-l { + margin-right: -4rem + } + + .swagger-ui .nr6-l { + margin-right: -8rem + } + + .swagger-ui .nr7-l { + margin-right: -16rem + } + + .swagger-ui .nb1-l { + margin-bottom: -.25rem + } + + .swagger-ui .nb2-l { + margin-bottom: -.5rem + } + + .swagger-ui .nb3-l { + margin-bottom: -1rem + } + + .swagger-ui .nb4-l { + margin-bottom: -2rem + } + + .swagger-ui .nb5-l { + margin-bottom: -4rem + } + + .swagger-ui .nb6-l { + margin-bottom: -8rem + } + + .swagger-ui .nb7-l { + margin-bottom: -16rem + } + + .swagger-ui .nt1-l { + margin-top: -.25rem + } + + .swagger-ui .nt2-l { + margin-top: -.5rem + } + + .swagger-ui .nt3-l { + margin-top: -1rem + } + + .swagger-ui .nt4-l { + margin-top: -2rem + } + + .swagger-ui .nt5-l { + margin-top: -4rem + } + + .swagger-ui .nt6-l { + margin-top: -8rem + } + + .swagger-ui .nt7-l { + margin-top: -16rem + } +} + +.swagger-ui .collapse { + border-collapse: collapse; + border-spacing: 0 +} + +.swagger-ui .striped--light-silver:nth-child(odd) { + background-color: #aaa +} + +.swagger-ui .striped--moon-gray:nth-child(odd) { + background-color: #ccc +} + +.swagger-ui .striped--light-gray:nth-child(odd) { + background-color: #eee +} + +.swagger-ui .striped--near-white:nth-child(odd) { + background-color: #f4f4f4 +} + +.swagger-ui .stripe-light:nth-child(odd) { + background-color: hsla(0,0%,100%,.1) +} + +.swagger-ui .stripe-dark:nth-child(odd) { + background-color: rgba(0,0,0,.1) +} + +.swagger-ui .strike { + -webkit-text-decoration: line-through; + text-decoration: line-through +} + +.swagger-ui .underline { + -webkit-text-decoration: underline; + text-decoration: underline +} + +.swagger-ui .no-underline { + -webkit-text-decoration: none; + text-decoration: none +} + +@media screen and (min-width: 30em) { + .swagger-ui .strike-ns { + -webkit-text-decoration:line-through; + text-decoration: line-through + } + + .swagger-ui .underline-ns { + -webkit-text-decoration: underline; + text-decoration: underline + } + + .swagger-ui .no-underline-ns { + -webkit-text-decoration: none; + text-decoration: none + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .strike-m { + -webkit-text-decoration:line-through; + text-decoration: line-through + } + + .swagger-ui .underline-m { + -webkit-text-decoration: underline; + text-decoration: underline + } + + .swagger-ui .no-underline-m { + -webkit-text-decoration: none; + text-decoration: none + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .strike-l { + -webkit-text-decoration:line-through; + text-decoration: line-through + } + + .swagger-ui .underline-l { + -webkit-text-decoration: underline; + text-decoration: underline + } + + .swagger-ui .no-underline-l { + -webkit-text-decoration: none; + text-decoration: none + } +} + +.swagger-ui .tl { + text-align: left +} + +.swagger-ui .tr { + text-align: right +} + +.swagger-ui .tc { + text-align: center +} + +.swagger-ui .tj { + text-align: justify +} + +@media screen and (min-width: 30em) { + .swagger-ui .tl-ns { + text-align:left + } + + .swagger-ui .tr-ns { + text-align: right + } + + .swagger-ui .tc-ns { + text-align: center + } + + .swagger-ui .tj-ns { + text-align: justify + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .tl-m { + text-align:left + } + + .swagger-ui .tr-m { + text-align: right + } + + .swagger-ui .tc-m { + text-align: center + } + + .swagger-ui .tj-m { + text-align: justify + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .tl-l { + text-align:left + } + + .swagger-ui .tr-l { + text-align: right + } + + .swagger-ui .tc-l { + text-align: center + } + + .swagger-ui .tj-l { + text-align: justify + } +} + +.swagger-ui .ttc { + text-transform: capitalize +} + +.swagger-ui .ttl { + text-transform: lowercase +} + +.swagger-ui .ttu { + text-transform: uppercase +} + +.swagger-ui .ttn { + text-transform: none +} + +@media screen and (min-width: 30em) { + .swagger-ui .ttc-ns { + text-transform:capitalize + } + + .swagger-ui .ttl-ns { + text-transform: lowercase + } + + .swagger-ui .ttu-ns { + text-transform: uppercase + } + + .swagger-ui .ttn-ns { + text-transform: none + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .ttc-m { + text-transform:capitalize + } + + .swagger-ui .ttl-m { + text-transform: lowercase + } + + .swagger-ui .ttu-m { + text-transform: uppercase + } + + .swagger-ui .ttn-m { + text-transform: none + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .ttc-l { + text-transform:capitalize + } + + .swagger-ui .ttl-l { + text-transform: lowercase + } + + .swagger-ui .ttu-l { + text-transform: uppercase + } + + .swagger-ui .ttn-l { + text-transform: none + } +} + +.swagger-ui .f-6,.swagger-ui .f-headline { + font-size: 6rem +} + +.swagger-ui .f-5,.swagger-ui .f-subheadline { + font-size: 5rem +} + +.swagger-ui .f1 { + font-size: 3rem +} + +.swagger-ui .f2 { + font-size: 2.25rem +} + +.swagger-ui .f3 { + font-size: 1.5rem +} + +.swagger-ui .f4 { + font-size: 1.25rem +} + +.swagger-ui .f5 { + font-size: 1rem +} + +.swagger-ui .f6 { + font-size: .875rem +} + +.swagger-ui .f7 { + font-size: .75rem +} + +@media screen and (min-width: 30em) { + .swagger-ui .f-6-ns,.swagger-ui .f-headline-ns { + font-size:6rem + } + + .swagger-ui .f-5-ns,.swagger-ui .f-subheadline-ns { + font-size: 5rem + } + + .swagger-ui .f1-ns { + font-size: 3rem + } + + .swagger-ui .f2-ns { + font-size: 2.25rem + } + + .swagger-ui .f3-ns { + font-size: 1.5rem + } + + .swagger-ui .f4-ns { + font-size: 1.25rem + } + + .swagger-ui .f5-ns { + font-size: 1rem + } + + .swagger-ui .f6-ns { + font-size: .875rem + } + + .swagger-ui .f7-ns { + font-size: .75rem + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .f-6-m,.swagger-ui .f-headline-m { + font-size:6rem + } + + .swagger-ui .f-5-m,.swagger-ui .f-subheadline-m { + font-size: 5rem + } + + .swagger-ui .f1-m { + font-size: 3rem + } + + .swagger-ui .f2-m { + font-size: 2.25rem + } + + .swagger-ui .f3-m { + font-size: 1.5rem + } + + .swagger-ui .f4-m { + font-size: 1.25rem + } + + .swagger-ui .f5-m { + font-size: 1rem + } + + .swagger-ui .f6-m { + font-size: .875rem + } + + .swagger-ui .f7-m { + font-size: .75rem + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .f-6-l,.swagger-ui .f-headline-l { + font-size:6rem + } + + .swagger-ui .f-5-l,.swagger-ui .f-subheadline-l { + font-size: 5rem + } + + .swagger-ui .f1-l { + font-size: 3rem + } + + .swagger-ui .f2-l { + font-size: 2.25rem + } + + .swagger-ui .f3-l { + font-size: 1.5rem + } + + .swagger-ui .f4-l { + font-size: 1.25rem + } + + .swagger-ui .f5-l { + font-size: 1rem + } + + .swagger-ui .f6-l { + font-size: .875rem + } + + .swagger-ui .f7-l { + font-size: .75rem + } +} + +.swagger-ui .measure { + max-width: 30em +} + +.swagger-ui .measure-wide { + max-width: 34em +} + +.swagger-ui .measure-narrow { + max-width: 20em +} + +.swagger-ui .indent { + margin-bottom: 0; + margin-top: 0; + text-indent: 1em +} + +.swagger-ui .small-caps { + font-feature-settings: "smcp"; + font-variant: small-caps +} + +.swagger-ui .truncate { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap +} + +@media screen and (min-width: 30em) { + .swagger-ui .measure-ns { + max-width:30em + } + + .swagger-ui .measure-wide-ns { + max-width: 34em + } + + .swagger-ui .measure-narrow-ns { + max-width: 20em + } + + .swagger-ui .indent-ns { + margin-bottom: 0; + margin-top: 0; + text-indent: 1em + } + + .swagger-ui .small-caps-ns { + font-feature-settings: "smcp"; + font-variant: small-caps + } + + .swagger-ui .truncate-ns { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .measure-m { + max-width:30em + } + + .swagger-ui .measure-wide-m { + max-width: 34em + } + + .swagger-ui .measure-narrow-m { + max-width: 20em + } + + .swagger-ui .indent-m { + margin-bottom: 0; + margin-top: 0; + text-indent: 1em + } + + .swagger-ui .small-caps-m { + font-feature-settings: "smcp"; + font-variant: small-caps + } + + .swagger-ui .truncate-m { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .measure-l { + max-width:30em + } + + .swagger-ui .measure-wide-l { + max-width: 34em + } + + .swagger-ui .measure-narrow-l { + max-width: 20em + } + + .swagger-ui .indent-l { + margin-bottom: 0; + margin-top: 0; + text-indent: 1em + } + + .swagger-ui .small-caps-l { + font-feature-settings: "smcp"; + font-variant: small-caps + } + + .swagger-ui .truncate-l { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap + } +} + +.swagger-ui .overflow-container { + overflow-y: scroll +} + +.swagger-ui .center { + margin-left: auto; + margin-right: auto +} + +.swagger-ui .mr-auto { + margin-right: auto +} + +.swagger-ui .ml-auto { + margin-left: auto +} + +@media screen and (min-width: 30em) { + .swagger-ui .center-ns { + margin-left:auto; + margin-right: auto + } + + .swagger-ui .mr-auto-ns { + margin-right: auto + } + + .swagger-ui .ml-auto-ns { + margin-left: auto + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .center-m { + margin-left:auto; + margin-right: auto + } + + .swagger-ui .mr-auto-m { + margin-right: auto + } + + .swagger-ui .ml-auto-m { + margin-left: auto + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .center-l { + margin-left:auto; + margin-right: auto + } + + .swagger-ui .mr-auto-l { + margin-right: auto + } + + .swagger-ui .ml-auto-l { + margin-left: auto + } +} + +.swagger-ui .clip { + position: fixed!important; + _position: absolute!important; + clip: rect(1px 1px 1px 1px); + clip: rect(1px,1px,1px,1px) +} + +@media screen and (min-width: 30em) { + .swagger-ui .clip-ns { + position:fixed!important; + _position: absolute!important; + clip: rect(1px 1px 1px 1px); + clip: rect(1px,1px,1px,1px) + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .clip-m { + position:fixed!important; + _position: absolute!important; + clip: rect(1px 1px 1px 1px); + clip: rect(1px,1px,1px,1px) + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .clip-l { + position:fixed!important; + _position: absolute!important; + clip: rect(1px 1px 1px 1px); + clip: rect(1px,1px,1px,1px) + } +} + +.swagger-ui .ws-normal { + white-space: normal +} + +.swagger-ui .nowrap { + white-space: nowrap +} + +.swagger-ui .pre { + white-space: pre +} + +@media screen and (min-width: 30em) { + .swagger-ui .ws-normal-ns { + white-space:normal + } + + .swagger-ui .nowrap-ns { + white-space: nowrap + } + + .swagger-ui .pre-ns { + white-space: pre + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .ws-normal-m { + white-space:normal + } + + .swagger-ui .nowrap-m { + white-space: nowrap + } + + .swagger-ui .pre-m { + white-space: pre + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .ws-normal-l { + white-space:normal + } + + .swagger-ui .nowrap-l { + white-space: nowrap + } + + .swagger-ui .pre-l { + white-space: pre + } +} + +.swagger-ui .v-base { + vertical-align: baseline +} + +.swagger-ui .v-mid { + vertical-align: middle +} + +.swagger-ui .v-top { + vertical-align: top +} + +.swagger-ui .v-btm { + vertical-align: bottom +} + +@media screen and (min-width: 30em) { + .swagger-ui .v-base-ns { + vertical-align:baseline + } + + .swagger-ui .v-mid-ns { + vertical-align: middle + } + + .swagger-ui .v-top-ns { + vertical-align: top + } + + .swagger-ui .v-btm-ns { + vertical-align: bottom + } +} + +@media screen and (min-width: 30em)and (max-width:60em) { + .swagger-ui .v-base-m { + vertical-align:baseline + } + + .swagger-ui .v-mid-m { + vertical-align: middle + } + + .swagger-ui .v-top-m { + vertical-align: top + } + + .swagger-ui .v-btm-m { + vertical-align: bottom + } +} + +@media screen and (min-width: 60em) { + .swagger-ui .v-base-l { + vertical-align:baseline + } + + .swagger-ui .v-mid-l { + vertical-align: middle + } + + .swagger-ui .v-top-l { + vertical-align: top + } + + .swagger-ui .v-btm-l { + vertical-align: bottom + } +} + +.swagger-ui .dim { + opacity: 1; + transition: opacity .15s ease-in +} + +.swagger-ui .dim:focus,.swagger-ui .dim:hover { + opacity: .5; + transition: opacity .15s ease-in +} + +.swagger-ui .dim:active { + opacity: .8; + transition: opacity .15s ease-out +} + +.swagger-ui .glow { + transition: opacity .15s ease-in +} + +.swagger-ui .glow:focus,.swagger-ui .glow:hover { + opacity: 1; + transition: opacity .15s ease-in +} + +.swagger-ui .hide-child .child { + opacity: 0; + transition: opacity .15s ease-in +} + +.swagger-ui .hide-child:active .child,.swagger-ui .hide-child:focus .child,.swagger-ui .hide-child:hover .child { + opacity: 1; + transition: opacity .15s ease-in +} + +.swagger-ui .underline-hover:focus,.swagger-ui .underline-hover:hover { + -webkit-text-decoration: underline; + text-decoration: underline +} + +.swagger-ui .grow { + -moz-osx-font-smoothing: grayscale; + backface-visibility: hidden; + transform: translateZ(0); + transition: transform .25s ease-out +} + +.swagger-ui .grow:focus,.swagger-ui .grow:hover { + transform: scale(1.05) +} + +.swagger-ui .grow:active { + transform: scale(.9) +} + +.swagger-ui .grow-large { + -moz-osx-font-smoothing: grayscale; + backface-visibility: hidden; + transform: translateZ(0); + transition: transform .25s ease-in-out +} + +.swagger-ui .grow-large:focus,.swagger-ui .grow-large:hover { + transform: scale(1.2) +} + +.swagger-ui .grow-large:active { + transform: scale(.95) +} + +.swagger-ui .pointer:hover { + cursor: pointer +} + +.swagger-ui .shadow-hover { + cursor: pointer; + position: relative; + transition: all .5s cubic-bezier(.165,.84,.44,1) +} + +.swagger-ui .shadow-hover:after { + content: ""; + height: 100%; + left: 0; + opacity: 0; + position: absolute; + top: 0; + transition: opacity .5s cubic-bezier(.165,.84,.44,1); + width: 100%; + z-index: -1 +} + +.swagger-ui .shadow-hover:focus:after,.swagger-ui .shadow-hover:hover:after { + opacity: 1 +} + +.swagger-ui .bg-animate,.swagger-ui .bg-animate:focus,.swagger-ui .bg-animate:hover { + transition: background-color .15s ease-in-out +} + +.swagger-ui .z-0 { + z-index: 0 +} + +.swagger-ui .z-1 { + z-index: 1 +} + +.swagger-ui .z-2 { + z-index: 2 +} + +.swagger-ui .z-3 { + z-index: 3 +} + +.swagger-ui .z-4 { + z-index: 4 +} + +.swagger-ui .z-5 { + z-index: 5 +} + +.swagger-ui .z-999 { + z-index: 999 +} + +.swagger-ui .z-9999 { + z-index: 9999 +} + +.swagger-ui .z-max { + z-index: 2147483647 +} + +.swagger-ui .z-inherit { + z-index: inherit +} + +.swagger-ui .z-initial,.swagger-ui .z-unset { + z-index: auto +} + +.swagger-ui .nested-copy-line-height ol,.swagger-ui .nested-copy-line-height p,.swagger-ui .nested-copy-line-height ul { + line-height: 1.5 +} + +.swagger-ui .nested-headline-line-height h1,.swagger-ui .nested-headline-line-height h2,.swagger-ui .nested-headline-line-height h3,.swagger-ui .nested-headline-line-height h4,.swagger-ui .nested-headline-line-height h5,.swagger-ui .nested-headline-line-height h6 { + line-height: 1.25 +} + +.swagger-ui .nested-list-reset ol,.swagger-ui .nested-list-reset ul { + list-style-type: none; + margin-left: 0; + padding-left: 0 +} + +.swagger-ui .nested-copy-indent p+p { + margin-bottom: 0; + margin-top: 0; + text-indent: .1em +} + +.swagger-ui .nested-copy-seperator p+p { + margin-top: 1.5em +} + +.swagger-ui .nested-img img { + display: block; + max-width: 100%; + width: 100% +} + +.swagger-ui .nested-links a { + color: #357edd; + transition: color .15s ease-in +} + +.swagger-ui .nested-links a:focus,.swagger-ui .nested-links a:hover { + color: #96ccff; + transition: color .15s ease-in +} + +.swagger-ui .wrapper { + box-sizing: border-box; + margin: 0 auto; + max-width: 1460px; + padding: 0 20px; + width: 100% +} + +.swagger-ui .opblock-tag-section { + display: flex; + flex-direction: column +} + +.swagger-ui .try-out.btn-group { + display: flex; + flex: .1 2 auto; + padding: 0 +} + +.swagger-ui .try-out__btn { + margin-left: 1.25rem +} + +.swagger-ui .opblock-tag { + align-items: center; + cursor: pointer; + display: none; + padding: 10px 20px 10px 10px; + transition: all .2s +} + +.swagger-ui .opblock-tag:hover { + background: rgba(0,0,0,.02) +} + +.swagger-ui .opblock-tag { + color: #3b4151; + font-family: sans-serif; + font-size: 24px; + margin: 0 0 5px +} + +.swagger-ui .opblock-tag.no-desc span { + flex: 1 +} + +.swagger-ui .opblock-tag svg { + transition: all .4s +} + +.swagger-ui .opblock-tag small { + color: #3b4151; + flex: 2; + font-family: sans-serif; + font-size: 14px; + font-weight: 400; + padding: 0 10px +} + +.swagger-ui .opblock-tag>div { + flex: 1 1 150px; + font-weight: 400; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap +} + +@media(max-width: 640px) { + .swagger-ui .opblock-tag small,.swagger-ui .opblock-tag>div { + flex:1 + } +} + +.swagger-ui .opblock-tag .info__externaldocs { + text-align: right +} + +.swagger-ui .parameter__type { + color: #3b4151; + font-family: monospace; + font-size: 12px; + font-weight: 600; + padding: 5px 0 +} + +.swagger-ui .parameter-controls { + margin-top: .75em +} + +.swagger-ui .examples__title { + display: block; + font-size: 1.1em; + font-weight: 700; + margin-bottom: .75em +} + +.swagger-ui .examples__section { + margin-top: 1.5em +} + +.swagger-ui .examples__section-header { + font-size: .9rem; + font-weight: 700; + margin-bottom: .5rem +} + +.swagger-ui .examples-select { + display: inline-block; + margin-bottom: .75em +} + +.swagger-ui .examples-select .examples-select-element { + width: 100% +} + +.swagger-ui .examples-select__section-label { + font-size: .9rem; + font-weight: 700; + margin-right: .5rem +} + +.swagger-ui .example__section { + margin-top: 1.5em +} + +.swagger-ui .example__section-header { + font-size: .9rem; + font-weight: 700; + margin-bottom: .5rem +} + +.swagger-ui .view-line-link { + cursor: pointer; + margin: 0 5px; + position: relative; + top: 3px; + transition: all .5s; + width: 20px +} + +.swagger-ui .opblock .tab-header { + display: flex; + flex: 1 +} + +.swagger-ui .opblock .tab-header .tab-item { + cursor: pointer; + padding: 0 40px +} + +.swagger-ui .opblock .tab-header .tab-item:first-of-type { + padding: 0 40px 0 0 +} + +.swagger-ui .opblock .tab-header .tab-item.active h4 span { + position: relative +} + +.swagger-ui .opblock .tab-header .tab-item.active h4 span:after { + background: grey; + bottom: -15px; + content: ""; + height: 4px; + left: 50%; + position: absolute; + transform: translateX(-50%); + width: 120% +} + +.swagger-ui .opblock .opblock-section-header { + align-items: center; + background: #ccc; + display: flex; + min-height: 50px; + padding: 8px 20px +} + +.swagger-ui .opblock .opblock-section-header>label { + align-items: center; + color: #3b4151; + display: flex; + font-family: sans-serif; + font-size: 12px; + font-weight: 700; + margin: 0 0 0 auto +} + +.swagger-ui .opblock .opblock-section-header>label>span { + padding: 0 10px 0 0 +} + +.swagger-ui .opblock .opblock-section-header h4 { + color: #3b4151; + flex: 1; + font-family: sans-serif; + font-size: 14px; + margin: 0 +} + +.swagger-ui .opblock .opblock-summary-method { + background: #000; + color: #fff; + font-family: sans-serif; + font-size: 14px; + font-weight: 700; + min-width: 80px; + padding: 6px 0; + text-align: center; + text-shadow: 0 1px 0 rgba(0,0,0,.1) +} + +@media(max-width: 768px) { + .swagger-ui .opblock .opblock-summary-method { + font-size:12px + } +} + +.swagger-ui .opblock .opblock-summary-operation-id,.swagger-ui .opblock .opblock-summary-path,.swagger-ui .opblock .opblock-summary-path__deprecated { + align-items: center; + color: #3b4151; + display: flex; + font-family: monospace; + font-size: 16px; + font-weight: 600; + word-break: break-word +} + +@media(max-width: 768px) { + .swagger-ui .opblock .opblock-summary-operation-id,.swagger-ui .opblock .opblock-summary-path,.swagger-ui .opblock .opblock-summary-path__deprecated { + font-size:12px + } +} + +.swagger-ui .opblock .opblock-summary-path { + flex-shrink: 1 +} + +@media(max-width: 640px) { + .swagger-ui .opblock .opblock-summary-path { + max-width:100% + } +} + +.swagger-ui .opblock .opblock-summary-path__deprecated { + -webkit-text-decoration: line-through; + text-decoration: line-through +} + +.swagger-ui .opblock .opblock-summary-operation-id { + font-size: 14px +} + +.swagger-ui .opblock .opblock-summary-description { + color: #3b4151; + font-family: sans-serif; + font-size: 13px; + word-break: break-word +} + +.swagger-ui .opblock .opblock-summary-path-description-wrapper { + align-items: center; + display: flex; + flex-direction: row; + flex-grow: 1; + flex-wrap: wrap; + gap: 0 10px; + padding: 0 10px +} + +@media(max-width: 550px) { + .swagger-ui .opblock .opblock-summary-path-description-wrapper { + align-items:flex-start; + flex-direction: column + } +} + +.swagger-ui .opblock .opblock-summary { + align-items: center; + cursor: pointer; + display: flex; + padding: 5px +} + +.swagger-ui .opblock .opblock-summary .view-line-link { + cursor: pointer; + margin: 0; + position: relative; + top: 2px; + transition: all .5s; + width: 0 +} + +.swagger-ui .opblock .opblock-summary:hover .view-line-link { + margin: 0 5px; + width: 18px +} + +.swagger-ui .opblock .opblock-summary:hover .view-line-link.copy-to-clipboard { + width: 24px +} + +.swagger-ui .opblock.opblock-post { + background: rgba(73,204,144,.1); +} + +.swagger-ui .opblock.opblock-post .opblock-summary-method { + background: darkmagenta; + box-shadow: 4px 4px 8px #444 +} + +.swagger-ui .opblock.opblock-post .tab-header .tab-item.active h4 span:after { + background: #49cc90 +} + +.swagger-ui .opblock.opblock-put { + background: rgba(252,161,48,.1); +} + +.swagger-ui .opblock.opblock-put .opblock-summary-method { + background: #fca130 +} + +.swagger-ui .opblock.opblock-put .tab-header .tab-item.active h4 span:after { + background: #fca130 +} + +.swagger-ui .opblock.opblock-delete { + background: rgba(249,62,62,.1); +} + +.swagger-ui .opblock.opblock-delete .opblock-summary-method { + background: #f93e3e +} + +.swagger-ui .opblock.opblock-delete .tab-header .tab-item.active h4 span:after { + background: #f93e3e +} + +.swagger-ui .opblock.opblock-get { + background: rgba(97,175,254,.1); +} + +.swagger-ui .opblock.opblock-get .opblock-summary-method { + background: teal; + box-shadow: 4px 4px 8px #444; +} + +.swagger-ui .opblock.opblock-get .tab-header .tab-item.active h4 span:after { + background: #61affe +} + +.swagger-ui .opblock.opblock-patch { + background: rgba(80,227,194,.1); +} + +.swagger-ui .opblock.opblock-patch .opblock-summary-method { + background: #50e3c2 +} + +.swagger-ui .opblock.opblock-patch .tab-header .tab-item.active h4 span:after { + background: #50e3c2 +} + +.swagger-ui .opblock.opblock-head .opblock-summary-method { + background: #9012fe; + box-shadow: 4px 4px 8px #444; +} + +.swagger-ui .opblock.opblock-head .tab-header .tab-item.active h4 span:after { + background: #9012fe +} + +.swagger-ui .opblock.opblock-options { + background: rgba(13,90,167,.1); +} + +.swagger-ui .opblock.opblock-options .opblock-summary-method { + background: #0d5aa7 +} + +.swagger-ui .opblock.opblock-options .tab-header .tab-item.active h4 span:after { + background: #0d5aa7 +} + +.swagger-ui .opblock.opblock-deprecated { + background: hsla(0,0%,92%,.1); + opacity: .6 +} + +.swagger-ui .opblock.opblock-deprecated .opblock-summary-method { + background: #ebebeb +} + +.swagger-ui .opblock.opblock-deprecated .tab-header .tab-item.active h4 span:after { + background: #ebebeb +} + +.swagger-ui .opblock .opblock-schemes { + padding: 8px 20px +} + +.swagger-ui .opblock .opblock-schemes .schemes-title { + padding: 0 10px 0 0 +} + +.swagger-ui .filter .operation-filter-input { + margin: 20px 0; + padding: 10px; + width: 100% +} + +.swagger-ui .download-url-wrapper .failed,.swagger-ui .filter .failed { + color: red +} + +.swagger-ui .download-url-wrapper .loading,.swagger-ui .filter .loading { + color: #aaa +} + +.swagger-ui .model-example { + margin-top: 1em +} + +.swagger-ui .model-example .model-container { + overflow-x: auto; + width: 100% +} + +.swagger-ui .model-example .model-container .model-hint:not(.model-hint--embedded) { + top: -1.15em +} + +.swagger-ui .tab { + display: flex; + list-style: none; + padding: 0; + margin: 0.1em; +} + +.swagger-ui .tab li { + color: #3b4151; + cursor: pointer; + font-family: sans-serif; + font-size: 12px; + min-width: 60px; + padding: 0 +} + +.swagger-ui .tab li:first-of-type { + padding-left: 0; + padding-right: 12px; + position: relative +} + +.swagger-ui .tab li:first-of-type:after { + background: rgba(0,0,0,.2); + content: ""; + height: 100%; + position: absolute; + right: 6px; + top: 0; + width: 1px +} + +.swagger-ui .tab li.active { + font-weight: 700 +} + +.swagger-ui .tab li button.tablinks { + background: none; + color: inherit; + font-family: inherit; + font-weight: inherit; + padding: 0 +} + +.swagger-ui .opblock-description-wrapper,.swagger-ui .opblock-external-docs-wrapper,.swagger-ui .opblock-title_normal { + color: #3b4151; + font-family: sans-serif; + font-size: 12px; + margin: 0 0 5px; + padding: 15px 20px +} + +.swagger-ui .opblock-description-wrapper h4,.swagger-ui .opblock-external-docs-wrapper h4,.swagger-ui .opblock-title_normal h4 { + color: #3b4151; + font-family: sans-serif; + font-size: 12px; + margin: 0 0 5px +} + +.swagger-ui .opblock-description-wrapper p,.swagger-ui .opblock-external-docs-wrapper p,.swagger-ui .opblock-title_normal p { + color: #3b4151; + font-family: sans-serif; + font-size: 14px; + margin: 0 +} + +.swagger-ui .opblock-external-docs-wrapper h4 { + padding-left: 0 +} + +.swagger-ui .execute-wrapper { + padding: 20px; + text-align: right +} + +.swagger-ui .execute-wrapper .btn { + padding: 8px 40px; + width: 100% +} + +.swagger-ui .body-param-options { + display: flex; + flex-direction: column +} + +.swagger-ui .body-param-options .body-param-edit { + padding: 10px 0 +} + +.swagger-ui .body-param-options label { + padding: 8px 0 +} + +.swagger-ui .body-param-options label select { + margin: 3px 0 0 +} + +.swagger-ui .responses-inner { + padding-left: 20px; +} + +.swagger-ui .responses-inner h4,.swagger-ui .responses-inner h5 { + color: #3b4151; + font-family: sans-serif; + font-size: 12px; + margin: 10px 0 5px +} + +.swagger-ui .responses-inner .curl { + max-height: 400px; + min-height: 6em; + overflow-y: auto +} + +.swagger-ui .response-col_status { + color: #3b4151; + font-family: sans-serif; + font-size: 14px +} + +.swagger-ui .response-col_status .response-undocumented { + color: #909090; + font-family: monospace; + font-size: 11px; + font-weight: 600 +} + +.swagger-ui .response-col_links { + color: #3b4151; + font-family: sans-serif; + font-size: 14px; + max-width: 40em; + padding-left: 2em +} + +.swagger-ui .response-col_links .response-undocumented { + color: #909090; + font-family: monospace; + font-size: 11px; + font-weight: 600 +} + +.swagger-ui .response-col_links .operation-link { + margin-bottom: 1.5em +} + +.swagger-ui .response-col_links .operation-link .description { + margin-bottom: .5em +} + +.swagger-ui .opblock-body .opblock-loading-animation { + display: block; + margin: 3em auto +} + +.swagger-ui .opblock-body pre.microlight { + background: #333; + font-size: 12px; + hyphens: auto; + margin: 0; + padding: 10px; + white-space: pre-wrap; + word-break: break-all; + word-break: break-word; + word-wrap: break-word; + color: #fff; + font-family: monospace; + font-weight: 600 +} + +.swagger-ui .opblock-body pre.microlight .headerline { + display: block +} + +.swagger-ui .highlight-code { + position: relative +} + +.swagger-ui .highlight-code>.microlight { + max-height: 400px; + min-height: 6em; + overflow-y: auto +} + +.swagger-ui .highlight-code>.microlight code { + white-space: pre-wrap!important; + word-break: break-all +} + +.swagger-ui .curl-command { + position: relative +} + +.swagger-ui .download-contents { + align-items: center; + background: #7d8293; + bottom: 10px; + color: #fff; + display: flex; + font-family: sans-serif; + font-size: 14px; + font-weight: 600; + height: 30px; + justify-content: center; + padding: 5px; + position: absolute; + right: 10px; + text-align: center +} + +.swagger-ui .scheme-container { + background: #fff; + margin: 0 0 20px; + padding: 30px 0 +} + +.swagger-ui .scheme-container .schemes { + align-items: flex-end; + display: flex; + flex-wrap: wrap; + gap: 10px; + justify-content: space-between +} + +.swagger-ui .scheme-container .schemes>.schemes-server-container { + display: flex; + flex-wrap: wrap; + gap: 10px +} + +.swagger-ui .scheme-container .schemes>.schemes-server-container>label { + color: #3b4151; + display: flex; + flex-direction: column; + font-family: sans-serif; + font-size: 12px; + font-weight: 700; + margin: -20px 15px 0 0 +} + +.swagger-ui .scheme-container .schemes>.schemes-server-container>label select { + min-width: 130px; + text-transform: uppercase +} + +.swagger-ui .scheme-container .schemes:not(:has(.schemes-server-container)) { + justify-content: flex-end +} + +.swagger-ui .scheme-container .schemes .auth-wrapper { + flex: none; + justify-content: start +} + +.swagger-ui .scheme-container .schemes .auth-wrapper .authorize { + display: flex; + flex-wrap: nowrap; + margin: 0; + padding-right: 20px +} + +.swagger-ui .loading-container { + align-items: center; + display: flex; + flex-direction: column; + justify-content: center; + margin-top: 1em; + min-height: 1px; + padding: 40px 0 60px +} + +.swagger-ui .loading-container .loading { + position: relative +} + +.swagger-ui .loading-container .loading:after { + color: #3b4151; + content: "loading"; + font-family: sans-serif; + font-size: 10px; + font-weight: 700; + left: 50%; + position: absolute; + text-transform: uppercase; + top: 50%; + transform: translate(-50%,-50%) +} + +.swagger-ui .loading-container .loading:before { + animation: rotation 1s linear infinite,opacity .5s; + backface-visibility: hidden; + content: ""; + display: block; + height: 60px; + left: 50%; + margin: -30px; + opacity: 1; + position: absolute; + top: 50%; + width: 60px +} + +@keyframes rotation { + to { + transform: rotate(1turn) + } +} + +.swagger-ui .response-controls { + display: none; +} + +.swagger-ui .response-control-media-type { + margin-right: 1em +} + +.swagger-ui .response-control-media-type__accept-message { + color: green; + font-size: .7em +} + +.swagger-ui .response-control-examples__title,.swagger-ui .response-control-media-type__title { + display: block; + font-size: .7em; + margin-bottom: .2em +} + +@keyframes blinker { + 50% { + opacity: 0 + } +} + +.swagger-ui .hidden { + display: none +} + +.swagger-ui .no-margin { + height: auto; + margin: 0; + padding: 0 +} + +.swagger-ui .float-right { + float: right +} + +.swagger-ui .svg-assets { + height: 0; + position: absolute; + width: 0 +} + +.swagger-ui section h3 { + color: #3b4151; + font-family: sans-serif +} + +.swagger-ui a.nostyle { + display: inline +} + +.swagger-ui a.nostyle,.swagger-ui a.nostyle:visited { + color: inherit; + cursor: pointer; + text-decoration: inherit +} + +.swagger-ui .fallback { + color: #aaa; + padding: 1em +} + +.swagger-ui .version-pragma { + height: 100%; + padding: 5em 0 +} + +.swagger-ui .version-pragma__message { + display: flex; + font-size: 1.2em; + height: 100%; + justify-content: center; + line-height: 1.5em; + padding: 0 .6em; + text-align: center +} + +.swagger-ui .version-pragma__message>div { + flex: 1; + max-width: 55ch +} + +.swagger-ui .version-pragma__message code { + background-color: #dedede; + padding: 4px 4px 2px; + white-space: pre +} + +.swagger-ui .opblock-link { + font-weight: 400 +} + +.swagger-ui .opblock-link.shown { + font-weight: 700 +} + +.swagger-ui span.token-string { + color: #555 +} + +.swagger-ui span.token-not-formatted { + color: #555; + font-weight: 700 +} + +.swagger-ui .btn { + background: transparent; + color: #3b4151; + font-family: sans-serif; + font-size: 14px; + font-weight: 700; + padding: 5px 23px; + transition: all .3s; + box-shadow: 4px 4px 8px #444 +} + +.swagger-ui .btn.btn-sm { + font-size: 12px; + padding: 4px 23px +} + +.swagger-ui .btn[disabled] { + cursor: not-allowed; + opacity: .3 +} + +.swagger-ui .btn.cancel { + background-color: transparent; + color: #ff6060; + font-family: sans-serif +} + +.swagger-ui .btn.authorize { + background-color: transparent; + color: #49cc90; + display: inline; + line-height: 1 +} + +.swagger-ui .btn.authorize span { + float: left; + padding: 4px 20px 0 0 +} + +.swagger-ui .btn.authorize svg { + fill: #49cc90 +} + +.swagger-ui .btn.execute { + background-color: #4990e2; + color: #fff +} + +.swagger-ui .btn-group { + display: flex; + padding: 10px +} + +.swagger-ui .btn-group .btn { + flex: 1 +} + +.swagger-ui .authorization__btn { + background: none; + padding: 0 0 0 10px +} + +.swagger-ui .authorization__btn .locked { + opacity: 1 +} + +.swagger-ui .authorization__btn .unlocked { + opacity: .4 +} + +.swagger-ui .model-box-control,.swagger-ui .models-control,.swagger-ui .opblock-summary-control { + all: inherit; + cursor: pointer; + flex: 1; + padding: 0 +} + +.swagger-ui .model-box-control:focus,.swagger-ui .models-control:focus,.swagger-ui .opblock-summary-control:focus { + outline: auto +} + +.swagger-ui .expand-methods,.swagger-ui .expand-operation { + background: none; +} + +.swagger-ui .expand-methods svg,.swagger-ui .expand-operation svg { + height: 20px; + width: 20px +} + +.swagger-ui .expand-methods { + padding: 0 10px +} + +.swagger-ui .expand-methods:hover svg { + fill: #404040 +} + +.swagger-ui .expand-methods svg { + transition: all .3s; + fill: #707070 +} + +.swagger-ui button { + cursor: pointer; + border: none; + box-shadow: 2px 2px 8px #aaa; +} + +.swagger-ui button.invalid { + animation: shake .4s 1; + background: #feebeb; +} + +.swagger-ui .copy-to-clipboard { + align-items: center; + background: #7d8293; + bottom: 10px; + display: flex; + height: 30px; + justify-content: center; + position: absolute; + right: 100px; + width: 30px +} + +.swagger-ui .copy-to-clipboard button { + background: url("data:image/svg+xml;charset=utf-8,") 50% no-repeat; + flex-grow: 1; + flex-shrink: 1; + height: 25px +} + +.swagger-ui .copy-to-clipboard:active { + background: #5e626f +} + +.swagger-ui .opblock-control-arrow { + background: none; + text-align: center +} + +.swagger-ui .curl-command .copy-to-clipboard { + bottom: 5px; + height: 20px; + right: 10px; + width: 20px +} + +.swagger-ui .curl-command .copy-to-clipboard button { + height: 18px +} + +.swagger-ui .opblock .opblock-summary .view-line-link.copy-to-clipboard { + height: 26px; + position: static +} + +.swagger-ui select { + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; + background: #f7f7f7 url("data:image/svg+xml;charset=utf-8,") right 10px center no-repeat; + background-size: 20px; + color: #3b4151; + font-family: sans-serif; + font-size: 14px; + font-weight: 700; + padding: 5px 40px 5px 10px; + border: none; +} + +.swagger-ui select[multiple] { + background: #f7f7f7; + margin: 5px 0; + padding: 5px +} + +.swagger-ui select.invalid { + animation: shake .4s 1; + background: #feebeb; +} + +.swagger-ui .opblock-body select { + min-width: 230px +} + +@media(max-width: 768px) { + .swagger-ui .opblock-body select { + min-width:180px + } +} + +@media(max-width: 640px) { + .swagger-ui .opblock-body select { + min-width:100%; + width: 100% + } +} + +.swagger-ui label { + color: #3b4151; + font-family: sans-serif; + font-size: 12px; + font-weight: 700; + margin: 0 0 5px +} + +.swagger-ui input[type=email],.swagger-ui input[type=file],.swagger-ui input[type=password],.swagger-ui input[type=search],.swagger-ui input[type=text] { + line-height: 1 +} + +@media(max-width: 768px) { + .swagger-ui input[type=email],.swagger-ui input[type=file],.swagger-ui input[type=password],.swagger-ui input[type=search],.swagger-ui input[type=text] { + max-width:175px + } +} + +.swagger-ui input[type=email],.swagger-ui input[type=file],.swagger-ui input[type=password],.swagger-ui input[type=search],.swagger-ui input[type=text],.swagger-ui textarea { + background: #fff; + margin: 5px 0; + min-width: 100px; + padding: 8px 10px +} + +.swagger-ui input[type=email].invalid,.swagger-ui input[type=file].invalid,.swagger-ui input[type=password].invalid,.swagger-ui input[type=search].invalid,.swagger-ui input[type=text].invalid,.swagger-ui textarea.invalid { + animation: shake .4s 1; + background: #feebeb; +} + +.swagger-ui input[disabled],.swagger-ui select[disabled],.swagger-ui textarea[disabled] { + background-color: #fafafa; + color: #888; + cursor: not-allowed +} + +.swagger-ui textarea[disabled] { + background-color: #41444e; + color: #fff +} + +@keyframes shake { + 10%,90% { + transform: translate3d(-1px,0,0) + } + + 20%,80% { + transform: translate3d(2px,0,0) + } + + 30%,50%,70% { + transform: translate3d(-4px,0,0) + } + + 40%,60% { + transform: translate3d(4px,0,0) + } +} + +.swagger-ui textarea { + background: hsla(0,0%,100%,.8); + color: #3b4151; + font-family: monospace; + font-size: 12px; + font-weight: 600; + min-height: 280px; + outline: none; + padding: 10px; + width: 100% +} + +.swagger-ui textarea.curl { + background: #41444e; + color: #fff; + font-family: monospace; + font-size: 12px; + font-weight: 600; + margin: 0; + min-height: 100px; + padding: 10px; + resize: none +} + +.swagger-ui .checkbox { + color: #303030; + padding: 5px 0 10px; + transition: opacity .5s +} + +.swagger-ui .checkbox label { + display: flex +} + +.swagger-ui .checkbox p { + color: #3b4151; + font-family: monospace; + font-style: italic; + font-weight: 400!important; + font-weight: 600; + margin: 0!important +} + +.swagger-ui .checkbox input[type=checkbox] { + display: none +} + +.swagger-ui .checkbox input[type=checkbox]+label>.item { + background: #e8e8e8; + cursor: pointer; + display: inline-block; + flex: none; + height: 16px; + margin: 0 8px 0 0; + padding: 5px; + position: relative; + top: 3px; + width: 16px +} + +.swagger-ui .checkbox input[type=checkbox]+label>.item:active { + transform: scale(.9) +} + +.swagger-ui .checkbox input[type=checkbox]:checked+label>.item { + background: #e8e8e8 url("data:image/svg+xml;charset=utf-8,") 50% no-repeat +} + +.swagger-ui .dialog-ux { + bottom: 0; + left: 0; + position: fixed; + right: 0; + top: 0; + z-index: 9999 +} + +.swagger-ui .dialog-ux .backdrop-ux { + background: rgba(0,0,0,.8); + bottom: 0; + left: 0; + position: fixed; + right: 0; + top: 0 +} + +.swagger-ui .dialog-ux .modal-ux { + background: #fff; + left: 50%; + max-width: 650px; + min-width: 300px; + position: absolute; + top: 50%; + transform: translate(-50%,-50%); + width: 100%; + z-index: 9999 +} + +.swagger-ui .dialog-ux .modal-ux-content { + max-height: 540px; + overflow-y: auto; + padding: 20px +} + +.swagger-ui .dialog-ux .modal-ux-content p { + color: #41444e; + color: #3b4151; + font-family: sans-serif; + font-size: 12px; + margin: 0 0 5px +} + +.swagger-ui .dialog-ux .modal-ux-content h4 { + color: #3b4151; + font-family: sans-serif; + font-size: 18px; + font-weight: 600; + margin: 15px 0 0 +} + +.swagger-ui .dialog-ux .modal-ux-header { + align-items: center; + display: flex; + padding: 12px 0 +} + +.swagger-ui .dialog-ux .modal-ux-header .close-modal { + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; + background: none; + padding: 0 10px +} + +.swagger-ui .dialog-ux .modal-ux-header h3 { + color: #3b4151; + flex: 1; + font-family: sans-serif; + font-size: 20px; + font-weight: 600; + margin: 0; + padding: 0 20px +} + +.swagger-ui .model { + color: #3b4151; + font-family: monospace; + font-size: 12px; + font-weight: 300; + font-weight: 600 +} + +.swagger-ui .model .deprecated span,.swagger-ui .model .deprecated td { + color: #a0a0a0!important +} + +.swagger-ui .model .deprecated>td:first-of-type { + -webkit-text-decoration: line-through; + text-decoration: line-through +} + +.swagger-ui .model-toggle { + cursor: pointer; + display: inline-block; + font-size: 10px; + margin: auto .3em; + position: relative; + top: 6px; + transform: rotate(90deg); + transform-origin: 50% 50%; + transition: transform .15s ease-in +} + +.swagger-ui .model-toggle.collapsed { + transform: rotate(0deg) +} + +.swagger-ui .model-toggle:after { + background: url("data:image/svg+xml;charset=utf-8,") 50% no-repeat; + background-size: 100%; + content: ""; + display: block; + height: 20px; + width: 20px +} + +.swagger-ui .model-jump-to-path { + cursor: pointer; + position: relative +} + +.swagger-ui .model-jump-to-path .view-line-link { + cursor: pointer; + position: absolute; + top: -.4em +} + +.swagger-ui .model-title { + position: relative +} + +.swagger-ui .model-title:hover .model-hint { + display: block +} + +.swagger-ui .model-hint { + background: rgba(0,0,0,.7); + color: #ebebeb; + display: none; + padding: .1em .5em; + position: absolute; + top: -1.8em; + white-space: nowrap +} + +.swagger-ui .model p { + margin: 0 0 1em +} + +.swagger-ui .model .property { + color: #999; + font-style: italic +} + +.swagger-ui .model .property.primitive { + color: #6b6b6b +} + +.swagger-ui .model .property.primitive.extension { + display: block +} + +.swagger-ui .model .property.primitive.extension>td:first-child { + padding-left: 0; + padding-right: 0; + width: auto +} + +.swagger-ui .model .property.primitive.extension>td:first-child:after { + content: ": " +} + +.swagger-ui .model .external-docs,.swagger-ui table.model tr.description { + color: #666; + font-weight: 400 +} + +.swagger-ui table.model tr.description td:first-child,.swagger-ui table.model tr.property-row.required td:first-child { + font-weight: 700 +} + +.swagger-ui table.model tr.property-row td { + vertical-align: top +} + +.swagger-ui table.model tr.property-row td:first-child { + padding-right: .2em +} + +.swagger-ui table.model tr.property-row .star { + color: red +} + +.swagger-ui table.model tr.extension { + color: #777 +} + +.swagger-ui table.model tr.extension td:last-child { + vertical-align: top +} + +.swagger-ui table.model tr.external-docs td:first-child { + font-weight: 700 +} + +.swagger-ui table.model tr .renderedMarkdown p:first-child { + margin-top: 0 +} + +.swagger-ui section.models { + margin: 30px 0 +} + +.swagger-ui section.models .pointer { + cursor: pointer +} + +.swagger-ui section.models.is-open { + padding: 0 0 20px +} + +.swagger-ui section.models.is-open h4 { + margin: 0 0 5px +} + +.swagger-ui section.models h4 { + align-items: center; + color: #606060; + cursor: pointer; + display: flex; + font-family: sans-serif; + font-size: 16px; + margin: 0; + padding: 10px 20px 10px 10px; + transition: all .2s +} + +.swagger-ui section.models h4 svg { + transition: all .4s +} + +.swagger-ui section.models h4 span { + flex: 1 +} + +.swagger-ui section.models h4:hover { + background: rgba(0,0,0,.02) +} + +.swagger-ui section.models h5 { + color: #707070; + font-family: sans-serif; + font-size: 16px; + margin: 0 0 10px +} + +.swagger-ui section.models .model-jump-to-path { + position: relative; + top: 5px +} + +.swagger-ui section.models .model-container { + background: rgba(0,0,0,.05); + margin: 0 20px 15px; + position: relative; + transition: all .5s +} + +.swagger-ui section.models .model-container:hover { + background: rgba(0,0,0,.07) +} + +.swagger-ui section.models .model-container:first-of-type { + margin: 20px +} + +.swagger-ui section.models .model-container:last-of-type { + margin: 0 20px +} + +.swagger-ui section.models .model-container .models-jump-to-path { + opacity: .65; + position: absolute; + right: 5px; + top: 8px +} + +.swagger-ui section.models .model-box { + background: none +} + +.swagger-ui section.models .model-box:has(.model-box) { + overflow-x: auto; + width: 100% +} + +.swagger-ui .model-box { + background: rgba(0,0,0,.1); + display: inline-block; + padding: 10px +} + +.swagger-ui .model-box .model-jump-to-path { + position: relative; + top: 4px +} + +.swagger-ui .model-box.deprecated { + opacity: .5 +} + +.swagger-ui .model-title { + color: #505050; + font-family: sans-serif; + font-size: 16px +} + +.swagger-ui .model-title img { + bottom: 0; + margin-left: 1em; + position: relative +} + +.swagger-ui .model-deprecated-warning { + color: #f93e3e; + font-family: sans-serif; + font-size: 16px; + font-weight: 600; + margin-right: 1em +} + +.swagger-ui span>span.model .brace-close { + padding: 0 0 0 10px +} + +.swagger-ui .prop-name { + display: inline-block; + margin-right: 1em +} + +.swagger-ui .prop-type { + color: #55a +} + +.swagger-ui .prop-enum { + display: block +} + +.swagger-ui .prop-format { + color: #606060 +} + +.swagger-ui .servers>label { + color: #3b4151; + font-family: sans-serif; + font-size: 12px; + margin: -20px 15px 0 0 +} + +.swagger-ui .servers>label select { + max-width: 100%; + min-width: 130px; + width: 100% +} + +.swagger-ui .servers h4.message { + padding-bottom: 2em +} + +.swagger-ui .servers table tr { + width: 30em +} + +.swagger-ui .servers table td { + display: inline-block; + max-width: 15em; + padding-bottom: 10px; + padding-top: 10px; + vertical-align: middle +} + +.swagger-ui .servers table td:first-of-type { + padding-right: 1em +} + +.swagger-ui .servers table td input { + height: 100%; + width: 100% +} + +.swagger-ui .servers .computed-url { + margin: 2em 0 +} + +.swagger-ui .servers .computed-url code { + display: inline-block; + font-size: 16px; + margin: 0 1em; + padding: 4px +} + +.swagger-ui .servers-title { + font-size: 12px; + font-weight: 700 +} + +.swagger-ui .operation-servers h4.message { + margin-bottom: 2em +} + +.swagger-ui table { + border-collapse: collapse; + padding: 0 10px; + width: 100% +} + +.swagger-ui table.model tbody tr td { + padding: 0 0 0 1em; + vertical-align: top +} + +.swagger-ui table.model tbody tr td:first-of-type { + padding: 0 0 0 2em; + width: 174px +} + +.swagger-ui table.headers td { + color: #3b4151; + font-family: monospace; + font-size: 12px; + font-weight: 300; + font-weight: 600; + vertical-align: middle +} + +.swagger-ui table.headers .header-example { + color: #999; + font-style: italic +} + +.swagger-ui table tbody tr td { + vertical-align: top +} + +.swagger-ui table tbody tr td:first-of-type { + min-width: 6em; + padding: 4px; +} + +.swagger-ui table tbody tr td:has(.model-box) { + max-width: 1px +} + +.swagger-ui table thead tr td,.swagger-ui table thead tr th { + color: #3b4151; + font-family: sans-serif; + font-size: 12px; + font-weight: 700; + padding: 12px 0; + text-align: left +} + +.swagger-ui .parameters-col_description { + margin-bottom: 2em; + width: 99% +} + +.swagger-ui .parameters-col_description input { + max-width: 340px; + width: 100%; + border: none; + background-color: #eee; +} + +.swagger-ui .parameters-col_description .markdown:first-child p:first-child,.swagger-ui .parameters-col_description .renderedMarkdown:first-child p:first-child { + margin: 0 +} + +.swagger-ui .parameter__name { + color: #3b4151; + font-family: sans-serif; + font-size: 16px; + font-weight: 400; + margin-right: .75em +} + +.swagger-ui .parameter__name.required { + font-weight: 700 +} + +.swagger-ui .parameter__name.required span { + color: red +} + +.swagger-ui .parameter__name.required:after { + color: rgba(255,0,0,.6); + content: "required"; + font-size: 10px; + padding: 5px; + position: relative; + top: -6px +} + +.swagger-ui .parameter__extension,.swagger-ui .parameter__in { + color: grey; + font-family: monospace; + font-size: 12px; + font-style: italic; + font-weight: 600 +} + +.swagger-ui .parameter__deprecated { + color: red; + font-family: monospace; + font-size: 12px; + font-style: italic; + font-weight: 600 +} + +.swagger-ui .parameter__empty_value_toggle { + display: block; + font-size: 13px; + padding-bottom: 12px; + padding-top: 5px +} + +.swagger-ui .parameter__empty_value_toggle input { + margin-right: 7px; + width: auto +} + +.swagger-ui .parameter__empty_value_toggle.disabled { + opacity: .7 +} + +.swagger-ui .table-container { + padding-left: 20px +} + +.swagger-ui .response-col_description { + width: 99% +} + +.swagger-ui .response-col_description .markdown p:first-child,.swagger-ui .response-col_description .renderedMarkdown p:first-child { + margin: 0 +} + +.swagger-ui .response-col_description .markdown p:last-child,.swagger-ui .response-col_description .renderedMarkdown p:last-child { + margin-bottom: 0 +} + +.swagger-ui .response-col_links { + min-width: 6em +} + +.swagger-ui .response__extension { + color: grey; + font-family: monospace; + font-size: 12px; + font-style: italic; + font-weight: 600 +} + +.swagger-ui .topbar { + background-color: #1b1b1b; + padding: 10px 0 +} + +.swagger-ui .topbar .topbar-wrapper { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: 10px +} + +@media(max-width: 550px) { + .swagger-ui .topbar .topbar-wrapper { + align-items:start; + flex-direction: column + } +} + +.swagger-ui .topbar a { + align-items: center; + color: #fff; + display: flex; + flex: 1; + font-family: sans-serif; + font-size: 1.5em; + font-weight: 700; + max-width: 300px; + -webkit-text-decoration: none; + text-decoration: none +} + +.swagger-ui .topbar a span { + margin: 0; + padding: 0 10px +} + +.swagger-ui .topbar .download-url-wrapper { + display: flex; + flex: 3; + justify-content: flex-end +} + +.swagger-ui .topbar .download-url-wrapper input[type=text] { + margin: 0; + max-width: 100%; + outline: none; + width: 100% +} + +.swagger-ui .topbar .download-url-wrapper .select-label { + align-items: center; + color: #f0f0f0; + display: flex; + margin: 0; + max-width: 600px; + width: 100% +} + +.swagger-ui .topbar .download-url-wrapper .select-label span { + flex: 1; + font-size: 16px; + padding: 0 10px 0 0; + text-align: right +} + +.swagger-ui .topbar .download-url-wrapper .select-label select { + flex: 2; + outline: none; + width: 100% +} + +.swagger-ui .topbar .download-url-wrapper .download-url-button { + background: #62a03f; + color: #fff; + font-family: sans-serif; + font-size: 16px; + font-weight: 700; + padding: 4px 30px +} + +@media(max-width: 550px) { + .swagger-ui .topbar .download-url-wrapper { + width:100% + } +} + +.swagger-ui .info { + margin: 20px 0; +} + +.swagger-ui .info.failed-config { + margin-left: auto; + margin-right: auto; + max-width: 880px; + text-align: center +} + +.swagger-ui .info hgroup.main a { + font-size: 12px +} + +.swagger-ui .info li,.swagger-ui .info p,.swagger-ui .info pre,.swagger-ui .info table { + font-size: 14px +} + +.swagger-ui .info h1,.swagger-ui .info h2,.swagger-ui .info h3,.swagger-ui .info h4,.swagger-ui .info h5,.swagger-ui .info li,.swagger-ui .info p,.swagger-ui .info table { + color: #3b4151; + font-family: sans-serif +} + +.swagger-ui .info a { + color: #4990e2; + font-family: sans-serif; + font-size: 14px; + transition: all .4s; + display: none; +} + +.swagger-ui .info a:hover { + color: #1f69c0 +} + +.swagger-ui .info>div { + margin: 0 0 5px +} + +.swagger-ui .info .base-url { + color: #3b4151; + font-family: monospace; + font-size: 12px; + font-weight: 300!important; + font-weight: 600; + margin: 0 +} + +.swagger-ui .info .title { + color: #3b4151; + font-family: sans-serif; + font-size: 36px; + margin: 0 +} + +.swagger-ui .info .title small { + background: #7d8492; + display: inline-block; + font-size: 10px; + margin: 0 0 0 5px; + padding: 2px 4px; + position: relative; + top: -5px; + vertical-align: super +} + +.swagger-ui .info .title small.version-stamp { + background-color: #89bf04 +} + +.swagger-ui .info .title small pre { + color: #fff; + font-family: sans-serif; + margin: 0; + padding: 0 +} + +.swagger-ui .auth-btn-wrapper { + display: flex; + justify-content: center; + padding: 10px 0 +} + +.swagger-ui .auth-btn-wrapper .btn-done { + margin-right: 1em +} + +.swagger-ui .auth-wrapper { + display: flex; + flex: 1; + justify-content: flex-end +} + +.swagger-ui .auth-wrapper .authorize { + margin-left: 10px; + margin-right: 10px; + padding-right: 20px +} + +.swagger-ui .auth-container { + margin: 0 0 10px; + padding: 10px 20px +} + +.swagger-ui .auth-container:last-of-type { + margin: 0; + padding: 10px 20px +} + +.swagger-ui .auth-container h4 { + margin: 5px 0 15px!important +} + +.swagger-ui .auth-container .wrapper { + margin: 0; + padding: 0 +} + +.swagger-ui .auth-container input[type=password],.swagger-ui .auth-container input[type=text] { + min-width: 230px +} + +.swagger-ui .auth-container .errors { + background-color: #fee; + color: red; + color: #3b4151; + font-family: monospace; + font-size: 12px; + font-weight: 600; + margin: 1em; + padding: 10px +} + +.swagger-ui .auth-container .errors b { + margin-right: 1em; + text-transform: capitalize +} + +.swagger-ui .scopes h2 { + color: #3b4151; + font-family: sans-serif; + font-size: 14px +} + +.swagger-ui .scopes h2 a { + color: #4990e2; + cursor: pointer; + font-size: 12px; + padding-left: 10px; + -webkit-text-decoration: underline; + text-decoration: underline +} + +.swagger-ui .scope-def { + padding: 0 0 20px +} + +.swagger-ui .errors-wrapper { + animation: scaleUp .5s; + background: rgba(249,62,62,.1); + margin: 20px; + padding: 10px 20px +} + +.swagger-ui .errors-wrapper .error-wrapper { + margin: 0 0 10px +} + +.swagger-ui .errors-wrapper .errors h4 { + color: #3b4151; + font-family: monospace; + font-size: 14px; + font-weight: 600; + margin: 0 +} + +.swagger-ui .errors-wrapper .errors small { + color: #606060 +} + +.swagger-ui .errors-wrapper .errors .message { + white-space: pre-line +} + +.swagger-ui .errors-wrapper .errors .message.thrown { + max-width: 100% +} + +.swagger-ui .errors-wrapper .errors .error-line { + cursor: pointer; + -webkit-text-decoration: underline; + text-decoration: underline +} + +.swagger-ui .errors-wrapper hgroup { + align-items: center; + display: flex +} + +.swagger-ui .errors-wrapper hgroup h4 { + color: #3b4151; + flex: 1; + font-family: sans-serif; + font-size: 20px; + margin: 0 +} + +@keyframes scaleUp { + 0% { + opacity: 0; + transform: scale(.8) + } + + to { + opacity: 1; + transform: scale(1) + } +} + +.swagger-ui .Resizer.vertical.disabled { + display: none +} + +.swagger-ui .markdown p,.swagger-ui .markdown pre,.swagger-ui .renderedMarkdown p,.swagger-ui .renderedMarkdown pre { + margin: 1em auto; + word-break: break-all; + word-break: break-word +} + +.swagger-ui .markdown pre,.swagger-ui .renderedMarkdown pre { + background: none; + color: #000; + font-weight: 400; + padding: 0; + white-space: pre-wrap +} + +.swagger-ui .markdown code,.swagger-ui .renderedMarkdown code { + background: rgba(0,0,0,.05); + color: #9012fe; + font-family: monospace; + font-size: 14px; + font-weight: 600; + padding: 5px 7px +} + +.swagger-ui .markdown pre>code,.swagger-ui .renderedMarkdown pre>code { + display: block +} + +.swagger-ui .json-schema-2020-12-keyword--\$vocabulary ul { + margin: 0 0 0 20px +} + +.swagger-ui .json-schema-2020-12-\$vocabulary-uri { + margin-left: 35px +} + +.swagger-ui .json-schema-2020-12-\$vocabulary-uri--disabled { + -webkit-text-decoration: line-through; + text-decoration: line-through +} + +.swagger-ui .json-schema-2020-12-keyword--const .json-schema-2020-12-json-viewer__name,.swagger-ui .json-schema-2020-12-keyword--const .json-schema-2020-12-json-viewer__value { + color: #3b4151; + font-style: normal +} + +.swagger-ui .json-schema-2020-12__constraint { + background-color: #805ad5; + color: #3b4151; + color: #fff; + font-family: monospace; + font-weight: 600; + line-height: 1.5; + margin-left: 10px; + padding: 1px 3px +} + +.swagger-ui .json-schema-2020-12__constraint--string { + background-color: #d69e2e; + color: #fff +} + +.swagger-ui .json-schema-2020-12-keyword--default .json-schema-2020-12-json-viewer__name,.swagger-ui .json-schema-2020-12-keyword--default .json-schema-2020-12-json-viewer__value { + color: #3b4151; + font-style: normal +} + +.swagger-ui .json-schema-2020-12-keyword--dependentRequired>ul { + display: inline-block; + margin: 0; + padding: 0 +} + +.swagger-ui .json-schema-2020-12-keyword--dependentRequired>ul li { + display: inline; + list-style-type: none +} + +.swagger-ui .json-schema-2020-12-keyword--description { + color: #6b6b6b; + font-size: 12px; + margin-left: 20px +} + +.swagger-ui .json-schema-2020-12-keyword--description p { + margin: 0 +} + +.swagger-ui .json-schema-2020-12-keyword--enum .json-schema-2020-12-json-viewer__name,.swagger-ui .json-schema-2020-12-keyword--enum .json-schema-2020-12-json-viewer__value,.swagger-ui .json-schema-2020-12-keyword--examples .json-schema-2020-12-json-viewer__name,.swagger-ui .json-schema-2020-12-keyword--examples .json-schema-2020-12-json-viewer__value { + color: #3b4151; + font-style: normal +} + +.swagger-ui .json-schema-2020-12-json-viewer-extension-keyword .json-schema-2020-12-json-viewer__name,.swagger-ui .json-schema-2020-12-json-viewer-extension-keyword .json-schema-2020-12-json-viewer__value { + color: #929292; + font-style: italic +} + +.swagger-ui .json-schema-2020-12-keyword--patternProperties ul { + margin: 0; + padding: 0 +} + +.swagger-ui .json-schema-2020-12-keyword--patternProperties .json-schema-2020-12__title:first-of-type:after,.swagger-ui .json-schema-2020-12-keyword--patternProperties .json-schema-2020-12__title:first-of-type:before { + color: #55a; + content: "/" +} + +.swagger-ui .json-schema-2020-12-keyword--properties>ul { + margin: 0; + padding: 0 +} + +.swagger-ui .json-schema-2020-12-property { + list-style-type: none +} + +.swagger-ui .json-schema-2020-12-property--required>.json-schema-2020-12:first-of-type>.json-schema-2020-12-head .json-schema-2020-12__title:after { + color: red; + content: "*"; + font-weight: 700 +} + +.swagger-ui .json-schema-2020-12__title { + color: #505050; + display: inline-block; + font-family: sans-serif; + font-size: 12px; + font-weight: 700; + line-height: normal +} + +.swagger-ui .json-schema-2020-12__title .json-schema-2020-12-keyword__name { + margin: 0 +} + +.swagger-ui .json-schema-2020-12-property { + margin: 7px 0 } .swagger-ui .json-schema-2020-12-property .json-schema-2020-12__title { - color: aqua; + color: #3b4151; + font-family: monospace; + font-size: 12px; + font-weight: 600; + vertical-align: middle } -@media only screen and (prefers-color-scheme: dark) { - - a { color: #8c8cfa; } - - ::-webkit-scrollbar-track-piece { background-color: rgba(255, 255, 255, .2) !important; } - - ::-webkit-scrollbar-track { background-color: rgba(255, 255, 255, .3) !important; } - - ::-webkit-scrollbar-thumb { background-color: rgba(255, 255, 255, .5) !important; } - - embed[type="application/pdf"] { filter: invert(90%); } - - html { - background: #1f1f1f !important; - box-sizing: border-box; - filter: contrast(100%) brightness(100%) saturate(100%); - overflow-y: scroll; - } - - body { - background: #1f1f1f; - background-color: #1f1f1f; - background-image: none !important; - } - - button, input, select, textarea { - background-color: #1f1f1f; - color: #bfbfbf; - } - - font, html { color: #bfbfbf; } - - .swagger-ui, .swagger-ui section h3 { color: #b5bac9; } - - .swagger-ui a { background-color: transparent; } - - .swagger-ui mark { - background-color: #664b00; - color: #bfbfbf; - } - - .swagger-ui legend { color: inherit; } - - .swagger-ui .debug * { outline: #e6da99 solid 1px; } - - .swagger-ui .debug-white * { outline: #fff solid 1px; } - - .swagger-ui .debug-black * { outline: #bfbfbf solid 1px; } - - .swagger-ui .debug-grid { background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMTExIDc5LjE1ODMyNSwgMjAxNS8wOS8xMC0wMToxMDoyMCAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6MTRDOTY4N0U2N0VFMTFFNjg2MzZDQjkwNkQ4MjgwMEIiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6MTRDOTY4N0Q2N0VFMTFFNjg2MzZDQjkwNkQ4MjgwMEIiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3NjcyQkQ3NjY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3NjcyQkQ3NzY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PsBS+GMAAAAjSURBVHjaYvz//z8DLsD4gcGXiYEAGBIKGBne//fFpwAgwAB98AaF2pjlUQAAAABJRU5ErkJggg==) 0 0; } - - .swagger-ui .debug-grid-16 { background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMTExIDc5LjE1ODMyNSwgMjAxNS8wOS8xMC0wMToxMDoyMCAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6ODYyRjhERDU2N0YyMTFFNjg2MzZDQjkwNkQ4MjgwMEIiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6ODYyRjhERDQ2N0YyMTFFNjg2MzZDQjkwNkQ4MjgwMEIiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3NjcyQkQ3QTY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3NjcyQkQ3QjY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PvCS01IAAABMSURBVHjaYmR4/5+BFPBfAMFm/MBgx8RAGWCn1AAmSg34Q6kBDKMGMDCwICeMIemF/5QawEipAWwUhwEjMDvbAWlWkvVBwu8vQIABAEwBCph8U6c0AAAAAElFTkSuQmCC) 0 0; } - - .swagger-ui .debug-grid-8-solid { background: url(data:image/jpeg;base64,/9j/4QAYRXhpZgAASUkqAAgAAAAAAAAAAAAAAP/sABFEdWNreQABAAQAAAAAAAD/4QMxaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wLwA8P3hwYWNrZXQgYmVnaW49Iu+7vyIgaWQ9Ilc1TTBNcENlaGlIenJlU3pOVGN6a2M5ZCI/PiA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJBZG9iZSBYTVAgQ29yZSA1LjYtYzExMSA3OS4xNTgzMjUsIDIwMTUvMDkvMTAtMDE6MTA6MjAgICAgICAgICI+IDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+IDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiIHhtbG5zOnhtcD0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wLyIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bXA6Q3JlYXRvclRvb2w9IkFkb2JlIFBob3Rvc2hvcCBDQyAyMDE1IChNYWNpbnRvc2gpIiB4bXBNTTpJbnN0YW5jZUlEPSJ4bXAuaWlkOkIxMjI0OTczNjdCMzExRTZCMkJDRTI0MDgxMDAyMTcxIiB4bXBNTTpEb2N1bWVudElEPSJ4bXAuZGlkOkIxMjI0OTc0NjdCMzExRTZCMkJDRTI0MDgxMDAyMTcxIj4gPHhtcE1NOkRlcml2ZWRGcm9tIHN0UmVmOmluc3RhbmNlSUQ9InhtcC5paWQ6QjEyMjQ5NzE2N0IzMTFFNkIyQkNFMjQwODEwMDIxNzEiIHN0UmVmOmRvY3VtZW50SUQ9InhtcC5kaWQ6QjEyMjQ5NzI2N0IzMTFFNkIyQkNFMjQwODEwMDIxNzEiLz4gPC9yZGY6RGVzY3JpcHRpb24+IDwvcmRmOlJERj4gPC94OnhtcG1ldGE+IDw/eHBhY2tldCBlbmQ9InIiPz7/7gAOQWRvYmUAZMAAAAAB/9sAhAAbGhopHSlBJiZBQi8vL0JHPz4+P0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHAR0pKTQmND8oKD9HPzU/R0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0f/wAARCAAIAAgDASIAAhEBAxEB/8QAWQABAQAAAAAAAAAAAAAAAAAAAAYBAQEAAAAAAAAAAAAAAAAAAAIEEAEBAAMBAAAAAAAAAAAAAAABADECA0ERAAEDBQAAAAAAAAAAAAAAAAARITFBUWESIv/aAAwDAQACEQMRAD8AoOnTV1QTD7JJshP3vSM3P//Z) 0 0 #1c1c21; } - - .swagger-ui .debug-grid-16-solid { background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMTExIDc5LjE1ODMyNSwgMjAxNS8wOS8xMC0wMToxMDoyMCAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6NzY3MkJEN0U2N0M1MTFFNkIyQkNFMjQwODEwMDIxNzEiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6NzY3MkJEN0Y2N0M1MTFFNkIyQkNFMjQwODEwMDIxNzEiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3NjcyQkQ3QzY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3NjcyQkQ3RDY3QzUxMUU2QjJCQ0UyNDA4MTAwMjE3MSIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/Pve6J3kAAAAzSURBVHjaYvz//z8D0UDsMwMjSRoYP5Gq4SPNbRjVMEQ1fCRDg+in/6+J1AJUxsgAEGAA31BAJMS0GYEAAAAASUVORK5CYII=) 0 0 #1c1c21; } - - .swagger-ui .b--black { border-color: #000; } - - .swagger-ui .b--near-black { border-color: #121212; } - - .swagger-ui .b--dark-gray { border-color: #333; } - - .swagger-ui .b--mid-gray { border-color: #545454; } - - .swagger-ui .b--gray { border-color: #787878; } - - .swagger-ui .b--silver { border-color: #999; } - - .swagger-ui .b--light-silver { border-color: #6e6e6e; } - - .swagger-ui .b--moon-gray { border-color: #4d4d4d; } - - .swagger-ui .b--light-gray { border-color: #2b2b2b; } - - .swagger-ui .b--near-white { border-color: #242424; } - - .swagger-ui .b--white { border-color: #1c1c21; } - - .swagger-ui .b--white-90 { border-color: rgba(28, 28, 33, .9); } - - .swagger-ui .b--white-80 { border-color: rgba(28, 28, 33, .8); } - - .swagger-ui .b--white-70 { border-color: rgba(28, 28, 33, .7); } - - .swagger-ui .b--white-60 { border-color: rgba(28, 28, 33, .6); } - - .swagger-ui .b--white-50 { border-color: rgba(28, 28, 33, .5); } - - .swagger-ui .b--white-40 { border-color: rgba(28, 28, 33, .4); } - - .swagger-ui .b--white-30 { border-color: rgba(28, 28, 33, .3); } - - .swagger-ui .b--white-20 { border-color: rgba(28, 28, 33, .2); } - - .swagger-ui .b--white-10 { border-color: rgba(28, 28, 33, .1); } - - .swagger-ui .b--white-05 { border-color: rgba(28, 28, 33, .05); } - - .swagger-ui .b--white-025 { border-color: rgba(28, 28, 33, .024); } - - .swagger-ui .b--white-0125 { border-color: rgba(28, 28, 33, .01); } - - .swagger-ui .b--black-90 { border-color: rgba(0, 0, 0, .9); } - - .swagger-ui .b--black-80 { border-color: rgba(0, 0, 0, .8); } - - .swagger-ui .b--black-70 { border-color: rgba(0, 0, 0, .7); } - - .swagger-ui .b--black-60 { border-color: rgba(0, 0, 0, .6); } - - .swagger-ui .b--black-50 { border-color: rgba(0, 0, 0, .5); } - - .swagger-ui .b--black-40 { border-color: rgba(0, 0, 0, .4); } - - .swagger-ui .b--black-30 { border-color: rgba(0, 0, 0, .3); } - - .swagger-ui .b--black-20 { border-color: rgba(0, 0, 0, .2); } - - .swagger-ui .b--black-10 { border-color: rgba(0, 0, 0, .1); } - - .swagger-ui .b--black-05 { border-color: rgba(0, 0, 0, .05); } - - .swagger-ui .b--black-025 { border-color: rgba(0, 0, 0, .024); } - - .swagger-ui .b--black-0125 { border-color: rgba(0, 0, 0, .01); } - - .swagger-ui .b--dark-red { border-color: #bc2f36; } - - .swagger-ui .b--red { border-color: #c83932; } - - .swagger-ui .b--light-red { border-color: #ab3c2b; } - - .swagger-ui .b--orange { border-color: #cc6e33; } - - .swagger-ui .b--purple { border-color: #5e2ca5; } - - .swagger-ui .b--light-purple { border-color: #672caf; } - - .swagger-ui .b--dark-pink { border-color: #ab2b81; } - - .swagger-ui .b--hot-pink { border-color: #c03086; } - - .swagger-ui .b--pink { border-color: #8f2464; } - - .swagger-ui .b--light-pink { border-color: #721d4d; } - - .swagger-ui .b--dark-green { border-color: #1c6e50; } - - .swagger-ui .b--green { border-color: #279b70; } - - .swagger-ui .b--light-green { border-color: #228762; } - - .swagger-ui .b--navy { border-color: #0d1d35; } - - .swagger-ui .b--dark-blue { border-color: #20497e; } - - .swagger-ui .b--blue { border-color: #4380d0; } - - .swagger-ui .b--light-blue { border-color: #20517e; } - - .swagger-ui .b--lightest-blue { border-color: #143a52; } - - .swagger-ui .b--washed-blue { border-color: #0c312d; } - - .swagger-ui .b--washed-green { border-color: #0f3d2c; } - - .swagger-ui .b--washed-red { border-color: #411010; } - - .swagger-ui .b--transparent { border-color: transparent; } - - .swagger-ui .b--gold, .swagger-ui .b--light-yellow, .swagger-ui .b--washed-yellow, .swagger-ui .b--yellow { border-color: #664b00; } - - .swagger-ui .shadow-1 { box-shadow: rgba(0, 0, 0, .2) 0 0 4px 2px; } - - .swagger-ui .shadow-2 { box-shadow: rgba(0, 0, 0, .2) 0 0 8px 2px; } - - .swagger-ui .shadow-3 { box-shadow: rgba(0, 0, 0, .2) 2px 2px 4px 2px; } - - .swagger-ui .shadow-4 { box-shadow: rgba(0, 0, 0, .2) 2px 2px 8px 0; } - - .swagger-ui .shadow-5 { box-shadow: rgba(0, 0, 0, .2) 4px 4px 8px 0; } - - @media screen and (min-width: 30em) { - .swagger-ui .shadow-1-ns { box-shadow: rgba(0, 0, 0, .2) 0 0 4px 2px; } - - .swagger-ui .shadow-2-ns { box-shadow: rgba(0, 0, 0, .2) 0 0 8px 2px; } - - .swagger-ui .shadow-3-ns { box-shadow: rgba(0, 0, 0, .2) 2px 2px 4px 2px; } - - .swagger-ui .shadow-4-ns { box-shadow: rgba(0, 0, 0, .2) 2px 2px 8px 0; } - - .swagger-ui .shadow-5-ns { box-shadow: rgba(0, 0, 0, .2) 4px 4px 8px 0; } - } - - @media screen and (max-width: 60em) and (min-width: 30em) { - .swagger-ui .shadow-1-m { box-shadow: rgba(0, 0, 0, .2) 0 0 4px 2px; } - - .swagger-ui .shadow-2-m { box-shadow: rgba(0, 0, 0, .2) 0 0 8px 2px; } - - .swagger-ui .shadow-3-m { box-shadow: rgba(0, 0, 0, .2) 2px 2px 4px 2px; } - - .swagger-ui .shadow-4-m { box-shadow: rgba(0, 0, 0, .2) 2px 2px 8px 0; } - - .swagger-ui .shadow-5-m { box-shadow: rgba(0, 0, 0, .2) 4px 4px 8px 0; } - } - - @media screen and (min-width: 60em) { - .swagger-ui .shadow-1-l { box-shadow: rgba(0, 0, 0, .2) 0 0 4px 2px; } - - .swagger-ui .shadow-2-l { box-shadow: rgba(0, 0, 0, .2) 0 0 8px 2px; } - - .swagger-ui .shadow-3-l { box-shadow: rgba(0, 0, 0, .2) 2px 2px 4px 2px; } - - .swagger-ui .shadow-4-l { box-shadow: rgba(0, 0, 0, .2) 2px 2px 8px 0; } - - .swagger-ui .shadow-5-l { box-shadow: rgba(0, 0, 0, .2) 4px 4px 8px 0; } - } - - .swagger-ui .black-05 { color: rgba(191, 191, 191, .05); } - - .swagger-ui .bg-black-05 { background-color: rgba(0, 0, 0, .05); } - - .swagger-ui .black-90, .swagger-ui .hover-black-90:focus, .swagger-ui .hover-black-90:hover { color: rgba(191, 191, 191, .9); } - - .swagger-ui .black-80, .swagger-ui .hover-black-80:focus, .swagger-ui .hover-black-80:hover { color: rgba(191, 191, 191, .8); } - - .swagger-ui .black-70, .swagger-ui .hover-black-70:focus, .swagger-ui .hover-black-70:hover { color: rgba(191, 191, 191, .7); } - - .swagger-ui .black-60, .swagger-ui .hover-black-60:focus, .swagger-ui .hover-black-60:hover { color: rgba(191, 191, 191, .6); } - - .swagger-ui .black-50, .swagger-ui .hover-black-50:focus, .swagger-ui .hover-black-50:hover { color: rgba(191, 191, 191, .5); } - - .swagger-ui .black-40, .swagger-ui .hover-black-40:focus, .swagger-ui .hover-black-40:hover { color: rgba(191, 191, 191, .4); } - - .swagger-ui .black-30, .swagger-ui .hover-black-30:focus, .swagger-ui .hover-black-30:hover { color: rgba(191, 191, 191, .3); } - - .swagger-ui .black-20, .swagger-ui .hover-black-20:focus, .swagger-ui .hover-black-20:hover { color: rgba(191, 191, 191, .2); } - - .swagger-ui .black-10, .swagger-ui .hover-black-10:focus, .swagger-ui .hover-black-10:hover { color: rgba(191, 191, 191, .1); } - - .swagger-ui .hover-white-90:focus, .swagger-ui .hover-white-90:hover, .swagger-ui .white-90 { color: rgba(255, 255, 255, .9); } - - .swagger-ui .hover-white-80:focus, .swagger-ui .hover-white-80:hover, .swagger-ui .white-80 { color: rgba(255, 255, 255, .8); } - - .swagger-ui .hover-white-70:focus, .swagger-ui .hover-white-70:hover, .swagger-ui .white-70 { color: rgba(255, 255, 255, .7); } - - .swagger-ui .hover-white-60:focus, .swagger-ui .hover-white-60:hover, .swagger-ui .white-60 { color: rgba(255, 255, 255, .6); } - - .swagger-ui .hover-white-50:focus, .swagger-ui .hover-white-50:hover, .swagger-ui .white-50 { color: rgba(255, 255, 255, .5); } - - .swagger-ui .hover-white-40:focus, .swagger-ui .hover-white-40:hover, .swagger-ui .white-40 { color: rgba(255, 255, 255, .4); } - - .swagger-ui .hover-white-30:focus, .swagger-ui .hover-white-30:hover, .swagger-ui .white-30 { color: rgba(255, 255, 255, .3); } - - .swagger-ui .hover-white-20:focus, .swagger-ui .hover-white-20:hover, .swagger-ui .white-20 { color: rgba(255, 255, 255, .2); } - - .swagger-ui .hover-white-10:focus, .swagger-ui .hover-white-10:hover, .swagger-ui .white-10 { color: rgba(255, 255, 255, .1); } - - .swagger-ui .hover-moon-gray:focus, .swagger-ui .hover-moon-gray:hover, .swagger-ui .moon-gray { color: #ccc; } - - .swagger-ui .hover-light-gray:focus, .swagger-ui .hover-light-gray:hover, .swagger-ui .light-gray { color: #ededed; } - - .swagger-ui .hover-near-white:focus, .swagger-ui .hover-near-white:hover, .swagger-ui .near-white { color: #f5f5f5; } - - .swagger-ui .dark-red, .swagger-ui .hover-dark-red:focus, .swagger-ui .hover-dark-red:hover { color: #e6999d; } - - .swagger-ui .hover-red:focus, .swagger-ui .hover-red:hover, .swagger-ui .red { color: #e69d99; } - - .swagger-ui .hover-light-red:focus, .swagger-ui .hover-light-red:hover, .swagger-ui .light-red { color: #e6a399; } - - .swagger-ui .hover-orange:focus, .swagger-ui .hover-orange:hover, .swagger-ui .orange { color: #e6b699; } - - .swagger-ui .gold, .swagger-ui .hover-gold:focus, .swagger-ui .hover-gold:hover { color: #e6d099; } - - .swagger-ui .hover-yellow:focus, .swagger-ui .hover-yellow:hover, .swagger-ui .yellow { color: #e6da99; } - - .swagger-ui .hover-light-yellow:focus, .swagger-ui .hover-light-yellow:hover, .swagger-ui .light-yellow { color: #ede6b6; } - - .swagger-ui .hover-purple:focus, .swagger-ui .hover-purple:hover, .swagger-ui .purple { color: #b99ae4; } - - .swagger-ui .hover-light-purple:focus, .swagger-ui .hover-light-purple:hover, .swagger-ui .light-purple { color: #bb99e6; } - - .swagger-ui .dark-pink, .swagger-ui .hover-dark-pink:focus, .swagger-ui .hover-dark-pink:hover { color: #e699cc; } - - .swagger-ui .hot-pink, .swagger-ui .hover-hot-pink:focus, .swagger-ui .hover-hot-pink:hover, .swagger-ui .hover-pink:focus, .swagger-ui .hover-pink:hover, .swagger-ui .pink { color: #e699c7; } - - .swagger-ui .hover-light-pink:focus, .swagger-ui .hover-light-pink:hover, .swagger-ui .light-pink { color: #edb6d5; } - - .swagger-ui .dark-green, .swagger-ui .green, .swagger-ui .hover-dark-green:focus, .swagger-ui .hover-dark-green:hover, .swagger-ui .hover-green:focus, .swagger-ui .hover-green:hover { color: #99e6c9; } - - .swagger-ui .hover-light-green:focus, .swagger-ui .hover-light-green:hover, .swagger-ui .light-green { color: #a1e8ce; } - - .swagger-ui .hover-navy:focus, .swagger-ui .hover-navy:hover, .swagger-ui .navy { color: #99b8e6; } - - .swagger-ui .blue, .swagger-ui .dark-blue, .swagger-ui .hover-blue:focus, .swagger-ui .hover-blue:hover, .swagger-ui .hover-dark-blue:focus, .swagger-ui .hover-dark-blue:hover { color: #99bae6; } - - .swagger-ui .hover-light-blue:focus, .swagger-ui .hover-light-blue:hover, .swagger-ui .light-blue { color: #a9cbea; } - - .swagger-ui .hover-lightest-blue:focus, .swagger-ui .hover-lightest-blue:hover, .swagger-ui .lightest-blue { color: #d6e9f5; } - - .swagger-ui .hover-washed-blue:focus, .swagger-ui .hover-washed-blue:hover, .swagger-ui .washed-blue { color: #f7fdfc; } - - .swagger-ui .hover-washed-green:focus, .swagger-ui .hover-washed-green:hover, .swagger-ui .washed-green { color: #ebfaf4; } - - .swagger-ui .hover-washed-yellow:focus, .swagger-ui .hover-washed-yellow:hover, .swagger-ui .washed-yellow { color: #fbf9ef; } - - .swagger-ui .hover-washed-red:focus, .swagger-ui .hover-washed-red:hover, .swagger-ui .washed-red { color: #f9e7e7; } - - .swagger-ui .color-inherit, .swagger-ui .hover-inherit:focus, .swagger-ui .hover-inherit:hover { color: inherit; } - - .swagger-ui .bg-black-90, .swagger-ui .hover-bg-black-90:focus, .swagger-ui .hover-bg-black-90:hover { background-color: rgba(0, 0, 0, .9); } - - .swagger-ui .bg-black-80, .swagger-ui .hover-bg-black-80:focus, .swagger-ui .hover-bg-black-80:hover { background-color: rgba(0, 0, 0, .8); } - - .swagger-ui .bg-black-70, .swagger-ui .hover-bg-black-70:focus, .swagger-ui .hover-bg-black-70:hover { background-color: rgba(0, 0, 0, .7); } - - .swagger-ui .bg-black-60, .swagger-ui .hover-bg-black-60:focus, .swagger-ui .hover-bg-black-60:hover { background-color: rgba(0, 0, 0, .6); } - - .swagger-ui .bg-black-50, .swagger-ui .hover-bg-black-50:focus, .swagger-ui .hover-bg-black-50:hover { background-color: rgba(0, 0, 0, .5); } - - .swagger-ui .bg-black-40, .swagger-ui .hover-bg-black-40:focus, .swagger-ui .hover-bg-black-40:hover { background-color: rgba(0, 0, 0, .4); } - - .swagger-ui .bg-black-30, .swagger-ui .hover-bg-black-30:focus, .swagger-ui .hover-bg-black-30:hover { background-color: rgba(0, 0, 0, .3); } - - .swagger-ui .bg-black-20, .swagger-ui .hover-bg-black-20:focus, .swagger-ui .hover-bg-black-20:hover { background-color: rgba(0, 0, 0, .2); } - - .swagger-ui .bg-white-90, .swagger-ui .hover-bg-white-90:focus, .swagger-ui .hover-bg-white-90:hover { background-color: rgba(28, 28, 33, .9); } - - .swagger-ui .bg-white-80, .swagger-ui .hover-bg-white-80:focus, .swagger-ui .hover-bg-white-80:hover { background-color: rgba(28, 28, 33, .8); } - - .swagger-ui .bg-white-70, .swagger-ui .hover-bg-white-70:focus, .swagger-ui .hover-bg-white-70:hover { background-color: rgba(28, 28, 33, .7); } - - .swagger-ui .bg-white-60, .swagger-ui .hover-bg-white-60:focus, .swagger-ui .hover-bg-white-60:hover { background-color: rgba(28, 28, 33, .6); } - - .swagger-ui .bg-white-50, .swagger-ui .hover-bg-white-50:focus, .swagger-ui .hover-bg-white-50:hover { background-color: rgba(28, 28, 33, .5); } - - .swagger-ui .bg-white-40, .swagger-ui .hover-bg-white-40:focus, .swagger-ui .hover-bg-white-40:hover { background-color: rgba(28, 28, 33, .4); } - - .swagger-ui .bg-white-30, .swagger-ui .hover-bg-white-30:focus, .swagger-ui .hover-bg-white-30:hover { background-color: rgba(28, 28, 33, .3); } - - .swagger-ui .bg-white-20, .swagger-ui .hover-bg-white-20:focus, .swagger-ui .hover-bg-white-20:hover { background-color: rgba(28, 28, 33, .2); } - - .swagger-ui .bg-black, .swagger-ui .hover-bg-black:focus, .swagger-ui .hover-bg-black:hover { background-color: #000; } - - .swagger-ui .bg-near-black, .swagger-ui .hover-bg-near-black:focus, .swagger-ui .hover-bg-near-black:hover { background-color: #121212; } - - .swagger-ui .bg-dark-gray, .swagger-ui .hover-bg-dark-gray:focus, .swagger-ui .hover-bg-dark-gray:hover { background-color: #333; } - - .swagger-ui .bg-mid-gray, .swagger-ui .hover-bg-mid-gray:focus, .swagger-ui .hover-bg-mid-gray:hover { background-color: #545454; } - - .swagger-ui .bg-gray, .swagger-ui .hover-bg-gray:focus, .swagger-ui .hover-bg-gray:hover { background-color: #787878; } - - .swagger-ui .bg-silver, .swagger-ui .hover-bg-silver:focus, .swagger-ui .hover-bg-silver:hover { background-color: #999; } - - .swagger-ui .bg-white, .swagger-ui .hover-bg-white:focus, .swagger-ui .hover-bg-white:hover { background-color: #1c1c21; } - - .swagger-ui .bg-transparent, .swagger-ui .hover-bg-transparent:focus, .swagger-ui .hover-bg-transparent:hover { background-color: transparent; } - - .swagger-ui .bg-dark-red, .swagger-ui .hover-bg-dark-red:focus, .swagger-ui .hover-bg-dark-red:hover { background-color: #bc2f36; } - - .swagger-ui .bg-red, .swagger-ui .hover-bg-red:focus, .swagger-ui .hover-bg-red:hover { background-color: #c83932; } - - .swagger-ui .bg-light-red, .swagger-ui .hover-bg-light-red:focus, .swagger-ui .hover-bg-light-red:hover { background-color: #ab3c2b; } - - .swagger-ui .bg-orange, .swagger-ui .hover-bg-orange:focus, .swagger-ui .hover-bg-orange:hover { background-color: #cc6e33; } - - .swagger-ui .bg-gold, .swagger-ui .bg-light-yellow, .swagger-ui .bg-washed-yellow, .swagger-ui .bg-yellow, .swagger-ui .hover-bg-gold:focus, .swagger-ui .hover-bg-gold:hover, .swagger-ui .hover-bg-light-yellow:focus, .swagger-ui .hover-bg-light-yellow:hover, .swagger-ui .hover-bg-washed-yellow:focus, .swagger-ui .hover-bg-washed-yellow:hover, .swagger-ui .hover-bg-yellow:focus, .swagger-ui .hover-bg-yellow:hover { background-color: #664b00; } - - .swagger-ui .bg-purple, .swagger-ui .hover-bg-purple:focus, .swagger-ui .hover-bg-purple:hover { background-color: #5e2ca5; } - - .swagger-ui .bg-light-purple, .swagger-ui .hover-bg-light-purple:focus, .swagger-ui .hover-bg-light-purple:hover { background-color: #672caf; } - - .swagger-ui .bg-dark-pink, .swagger-ui .hover-bg-dark-pink:focus, .swagger-ui .hover-bg-dark-pink:hover { background-color: #ab2b81; } - - .swagger-ui .bg-hot-pink, .swagger-ui .hover-bg-hot-pink:focus, .swagger-ui .hover-bg-hot-pink:hover { background-color: #c03086; } - - .swagger-ui .bg-pink, .swagger-ui .hover-bg-pink:focus, .swagger-ui .hover-bg-pink:hover { background-color: #8f2464; } - - .swagger-ui .bg-light-pink, .swagger-ui .hover-bg-light-pink:focus, .swagger-ui .hover-bg-light-pink:hover { background-color: #721d4d; } - - .swagger-ui .bg-dark-green, .swagger-ui .hover-bg-dark-green:focus, .swagger-ui .hover-bg-dark-green:hover { background-color: #1c6e50; } - - .swagger-ui .bg-green, .swagger-ui .hover-bg-green:focus, .swagger-ui .hover-bg-green:hover { background-color: #279b70; } - - .swagger-ui .bg-light-green, .swagger-ui .hover-bg-light-green:focus, .swagger-ui .hover-bg-light-green:hover { background-color: #228762; } - - .swagger-ui .bg-navy, .swagger-ui .hover-bg-navy:focus, .swagger-ui .hover-bg-navy:hover { background-color: #0d1d35; } - - .swagger-ui .bg-dark-blue, .swagger-ui .hover-bg-dark-blue:focus, .swagger-ui .hover-bg-dark-blue:hover { background-color: #20497e; } - - .swagger-ui .bg-blue, .swagger-ui .hover-bg-blue:focus, .swagger-ui .hover-bg-blue:hover { background-color: #4380d0; } - - .swagger-ui .bg-light-blue, .swagger-ui .hover-bg-light-blue:focus, .swagger-ui .hover-bg-light-blue:hover { background-color: #20517e; } - - .swagger-ui .bg-lightest-blue, .swagger-ui .hover-bg-lightest-blue:focus, .swagger-ui .hover-bg-lightest-blue:hover { background-color: #143a52; } - - .swagger-ui .bg-washed-blue, .swagger-ui .hover-bg-washed-blue:focus, .swagger-ui .hover-bg-washed-blue:hover { background-color: #0c312d; } - - .swagger-ui .bg-washed-green, .swagger-ui .hover-bg-washed-green:focus, .swagger-ui .hover-bg-washed-green:hover { background-color: #0f3d2c; } - - .swagger-ui .bg-washed-red, .swagger-ui .hover-bg-washed-red:focus, .swagger-ui .hover-bg-washed-red:hover { background-color: #411010; } - - .swagger-ui .bg-inherit, .swagger-ui .hover-bg-inherit:focus, .swagger-ui .hover-bg-inherit:hover { background-color: inherit; } - - .swagger-ui .shadow-hover { transition: all .5s cubic-bezier(.165, .84, .44, 1) 0s; } - - .swagger-ui .shadow-hover::after { - border-radius: inherit; - box-shadow: rgba(0, 0, 0, .2) 0 0 16px 2px; - content: ""; - height: 100%; - left: 0; - opacity: 0; - position: absolute; - top: 0; - transition: opacity .5s cubic-bezier(.165, .84, .44, 1) 0s; - width: 100%; - z-index: -1; - } - - .swagger-ui .bg-animate, .swagger-ui .bg-animate:focus, .swagger-ui .bg-animate:hover { transition: background-color .15s ease-in-out 0s; } - - .swagger-ui .nested-links a { - color: #99bae6; - transition: color .15s ease-in 0s; - } - - .swagger-ui .nested-links a:focus, .swagger-ui .nested-links a:hover { - color: #a9cbea; - transition: color .15s ease-in 0s; - } - - .swagger-ui .opblock-tag { - border-bottom: 1px solid rgba(58, 64, 80, .3); - color: #b5bac9; - transition: all .2s ease 0s; - } - - .swagger-ui .opblock-tag svg, .swagger-ui section.models h4 svg { transition: all .4s ease 0s; } - - .swagger-ui .opblock { - border: 1px solid #000; - border-radius: 4px; - box-shadow: rgba(0, 0, 0, .19) 0 0 3px; - margin: 0 0 15px; - } - - .swagger-ui .opblock .tab-header .tab-item.active h4 span::after { background: gray; } - - .swagger-ui .opblock.is-open .opblock-summary { border-bottom: 1px solid #000; } - - .swagger-ui .opblock .opblock-section-header { - background: rgba(28, 28, 33, .8); - box-shadow: rgba(0, 0, 0, .1) 0 1px 2px; - } - - .swagger-ui .opblock .opblock-section-header > label > span { padding: 0 10px 0 0; } - - .swagger-ui .opblock .opblock-summary-method { - background: #000; - color: #fff; - text-shadow: rgba(0, 0, 0, .1) 0 1px 0; - } - - .swagger-ui .opblock.opblock-post { - background: rgba(72, 203, 144, .1); - border-color: #48cb90; - } - - .swagger-ui .opblock.opblock-post .opblock-summary-method, .swagger-ui .opblock.opblock-post .tab-header .tab-item.active h4 span::after { background: #48cb90; } - - .swagger-ui .opblock.opblock-post .opblock-summary { border-color: #48cb90; } - - .swagger-ui .opblock.opblock-put { - background: rgba(213, 157, 88, .1); - border-color: #d59d58; - } - - .swagger-ui .opblock.opblock-put .opblock-summary-method, .swagger-ui .opblock.opblock-put .tab-header .tab-item.active h4 span::after { background: #d59d58; } - - .swagger-ui .opblock.opblock-put .opblock-summary { border-color: #d59d58; } - - .swagger-ui .opblock.opblock-delete { - background: rgba(200, 50, 50, .1); - border-color: #c83232; - } - - .swagger-ui .opblock.opblock-delete .opblock-summary-method, .swagger-ui .opblock.opblock-delete .tab-header .tab-item.active h4 span::after { background: #c83232; } - - .swagger-ui .opblock.opblock-delete .opblock-summary { border-color: #c83232; } - - .swagger-ui .opblock.opblock-get { - background: rgba(42, 105, 167, .1); - border-color: #2a69a7; - } - - .swagger-ui .opblock.opblock-get .opblock-summary-method, .swagger-ui .opblock.opblock-get .tab-header .tab-item.active h4 span::after { background: #2a69a7; } - - .swagger-ui .opblock.opblock-get .opblock-summary { border-color: #2a69a7; } - - .swagger-ui .opblock.opblock-patch { - background: rgba(92, 214, 188, .1); - border-color: #5cd6bc; - } - - .swagger-ui .opblock.opblock-patch .opblock-summary-method, .swagger-ui .opblock.opblock-patch .tab-header .tab-item.active h4 span::after { background: #5cd6bc; } - - .swagger-ui .opblock.opblock-patch .opblock-summary { border-color: #5cd6bc; } - - .swagger-ui .opblock.opblock-head { - background: rgba(140, 63, 207, .1); - border-color: #8c3fcf; - } - - .swagger-ui .opblock.opblock-head .opblock-summary-method, .swagger-ui .opblock.opblock-head .tab-header .tab-item.active h4 span::after { background: #8c3fcf; } - - .swagger-ui .opblock.opblock-head .opblock-summary { border-color: #8c3fcf; } - - .swagger-ui .opblock.opblock-options { - background: rgba(36, 89, 143, .1); - border-color: #24598f; - } - - .swagger-ui .opblock.opblock-options .opblock-summary-method, .swagger-ui .opblock.opblock-options .tab-header .tab-item.active h4 span::after { background: #24598f; } - - .swagger-ui .opblock.opblock-options .opblock-summary { border-color: #24598f; } - - .swagger-ui .opblock.opblock-deprecated { - background: rgba(46, 46, 46, .1); - border-color: #2e2e2e; - opacity: .6; - } - - .swagger-ui .opblock.opblock-deprecated .opblock-summary-method, .swagger-ui .opblock.opblock-deprecated .tab-header .tab-item.active h4 span::after { background: #2e2e2e; } - - .swagger-ui .opblock.opblock-deprecated .opblock-summary { border-color: #2e2e2e; } - - .swagger-ui .filter .operation-filter-input { border: 2px solid #2b3446; } - - .swagger-ui .tab li:first-of-type::after { background: rgba(0, 0, 0, .2); } - - .swagger-ui .download-contents { - background: #7c8192; - color: #fff; - } - - .swagger-ui .scheme-container { - background: #1c1c21; - box-shadow: rgba(0, 0, 0, .15) 0 1px 2px 0; - } - - .swagger-ui .loading-container .loading::before { - animation: 1s linear 0s infinite normal none running rotation, .5s ease 0s 1 normal none running opacity; - border-color: rgba(0, 0, 0, .6) rgba(84, 84, 84, .1) rgba(84, 84, 84, .1); - } - - .swagger-ui .response-control-media-type--accept-controller select { border-color: #196619; } - - .swagger-ui .response-control-media-type__accept-message { color: #99e699; } - - .swagger-ui .version-pragma__message code { background-color: #3b3b3b; } - - .swagger-ui .btn { - background: 0 0; - border: 2px solid gray; - box-shadow: rgba(0, 0, 0, .1) 0 1px 2px; - color: #b5bac9; - } - - .swagger-ui .btn:hover { box-shadow: rgba(0, 0, 0, .3) 0 0 5px; } - - .swagger-ui .btn.authorize, .swagger-ui .btn.cancel { - background-color: transparent; - border-color: #a72a2a; - color: #e69999; - } - - .swagger-ui .btn.cancel:hover { - background-color: #a72a2a; - color: #fff; - } - - .swagger-ui .btn.authorize { - border-color: #48cb90; - color: #9ce3c3; - } - - .swagger-ui .btn.authorize svg { fill: #9ce3c3; } - - .btn.authorize.unlocked:hover { - background-color: #48cb90; - color: #fff; - } - - .btn.authorize.unlocked:hover svg { - fill: #fbfbfb; - } - - .swagger-ui .btn.execute { - background-color: #5892d5; - border-color: #5892d5; - color: #fff; - } - - .swagger-ui .copy-to-clipboard { background: #7c8192; } - - .swagger-ui .copy-to-clipboard button { background: url("data:image/svg+xml;charset=utf-8,") 50% center no-repeat; } - - .swagger-ui select { - background: url("data:image/svg+xml;charset=utf-8,") right 10px center/20px no-repeat #212121; - background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50cy8xLjEvIgogICB4bWxuczpjYz0iaHR0cDovL2NyZWF0aXZlY29tbW9ucy5vcmcvbnMjIgogICB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFREL3NvZGlwb2RpLTAuZHRkIgogICB4bWxuczppbmtzY2FwZT0iaHR0cDovL3d3dy5pbmtzY2FwZS5vcmcvbmFtZXNwYWNlcy9pbmtzY2FwZSIKICAgaW5rc2NhcGU6dmVyc2lvbj0iMS4wICg0MDM1YTRmYjQ5LCAyMDIwLTA1LTAxKSIKICAgc29kaXBvZGk6ZG9jbmFtZT0iZG93bmxvYWQuc3ZnIgogICBpZD0ic3ZnNCIKICAgdmVyc2lvbj0iMS4xIgogICB2aWV3Qm94PSIwIDAgMjAgMjAiPgogIDxtZXRhZGF0YQogICAgIGlkPSJtZXRhZGF0YTEwIj4KICAgIDxyZGY6UkRGPgogICAgICA8Y2M6V29yawogICAgICAgICByZGY6YWJvdXQ9IiI+CiAgICAgICAgPGRjOmZvcm1hdD5pbWFnZS9zdmcreG1sPC9kYzpmb3JtYXQ+CiAgICAgICAgPGRjOnR5cGUKICAgICAgICAgICByZGY6cmVzb3VyY2U9Imh0dHA6Ly9wdXJsLm9yZy9kYy9kY21pdHlwZS9TdGlsbEltYWdlIiAvPgogICAgICA8L2NjOldvcms+CiAgICA8L3JkZjpSREY+CiAgPC9tZXRhZGF0YT4KICA8ZGVmcwogICAgIGlkPSJkZWZzOCIgLz4KICA8c29kaXBvZGk6bmFtZWR2aWV3CiAgICAgaW5rc2NhcGU6Y3VycmVudC1sYXllcj0ic3ZnNCIKICAgICBpbmtzY2FwZTp3aW5kb3ctbWF4aW1pemVkPSIxIgogICAgIGlua3NjYXBlOndpbmRvdy15PSItOSIKICAgICBpbmtzY2FwZTp3aW5kb3cteD0iLTkiCiAgICAgaW5rc2NhcGU6Y3k9IjEwIgogICAgIGlua3NjYXBlOmN4PSIxMCIKICAgICBpbmtzY2FwZTp6b29tPSI0MS41IgogICAgIHNob3dncmlkPSJmYWxzZSIKICAgICBpZD0ibmFtZWR2aWV3NiIKICAgICBpbmtzY2FwZTp3aW5kb3ctaGVpZ2h0PSIxMDAxIgogICAgIGlua3NjYXBlOndpbmRvdy13aWR0aD0iMTkyMCIKICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIgogICAgIGlua3NjYXBlOnBhZ2VvcGFjaXR5PSIwIgogICAgIGd1aWRldG9sZXJhbmNlPSIxMCIKICAgICBncmlkdG9sZXJhbmNlPSIxMCIKICAgICBvYmplY3R0b2xlcmFuY2U9IjEwIgogICAgIGJvcmRlcm9wYWNpdHk9IjEiCiAgICAgYm9yZGVyY29sb3I9IiM2NjY2NjYiCiAgICAgcGFnZWNvbG9yPSIjZmZmZmZmIiAvPgogIDxwYXRoCiAgICAgc3R5bGU9ImZpbGw6I2ZmZmZmZiIKICAgICBpZD0icGF0aDIiCiAgICAgZD0iTTEzLjQxOCA3Ljg1OWEuNjk1LjY5NSAwIDAxLjk3OCAwIC42OC42OCAwIDAxMCAuOTY5bC0zLjkwOCAzLjgzYS42OTcuNjk3IDAgMDEtLjk3OSAwbC0zLjkwOC0zLjgzYS42OC42OCAwIDAxMC0uOTY5LjY5NS42OTUgMCAwMS45NzggMEwxMCAxMWwzLjQxOC0zLjE0MXoiIC8+Cjwvc3ZnPgo=) right 10px center/20px no-repeat #1c1c21; - border: 2px solid #41444e; - } - - .swagger-ui select[multiple] { background: #212121; } - - .swagger-ui button.invalid, .swagger-ui input[type=email].invalid, .swagger-ui input[type=file].invalid, .swagger-ui input[type=password].invalid, .swagger-ui input[type=search].invalid, .swagger-ui input[type=text].invalid, .swagger-ui select.invalid, .swagger-ui textarea.invalid { - background: #390e0e; - border-color: #c83232; - } - - .swagger-ui input[type=email], .swagger-ui input[type=file], .swagger-ui input[type=password], .swagger-ui input[type=search], .swagger-ui input[type=text], .swagger-ui textarea { - background: #1c1c21; - border: 1px solid #404040; - } - - .swagger-ui textarea { - background: rgba(28, 28, 33, .8); - color: #b5bac9; - } - - .swagger-ui input[disabled], .swagger-ui select[disabled] { - background-color: #1f1f1f; - color: #bfbfbf; - } - - .swagger-ui textarea[disabled] { - background-color: #41444e; - color: #fff; - } - - .swagger-ui select[disabled] { border-color: #878787; } - - .swagger-ui textarea:focus { border: 2px solid #2a69a7; } - - .swagger-ui .checkbox input[type=checkbox] + label > .item { - background: #303030; - box-shadow: #303030 0 0 0 2px; - } - - .swagger-ui .checkbox input[type=checkbox]:checked + label > .item { background: url("data:image/svg+xml;charset=utf-8,") 50% center no-repeat #303030; } - - .swagger-ui .dialog-ux .backdrop-ux { background: rgba(0, 0, 0, .8); } - - .swagger-ui .dialog-ux .modal-ux { - background: #1c1c21; - border: 1px solid #2e2e2e; - box-shadow: rgba(0, 0, 0, .2) 0 10px 30px 0; - } - - .swagger-ui .dialog-ux .modal-ux-header .close-modal { background: 0 0; } - - .swagger-ui .model .deprecated span, .swagger-ui .model .deprecated td { color: #bfbfbf !important; } - - .swagger-ui .model-toggle::after { background: url("data:image/svg+xml;charset=utf-8,") 50% center/100% no-repeat; } - - .swagger-ui .model-hint { - background: rgba(0, 0, 0, .7); - color: #ebebeb; - } - - .swagger-ui section.models { border: 1px solid rgba(58, 64, 80, .3); } - - .swagger-ui section.models.is-open h4 { border-bottom: 1px solid rgba(58, 64, 80, .3); } - - .swagger-ui section.models .model-container { background: rgba(0, 0, 0, .05); } - - .swagger-ui section.models .model-container:hover { background: rgba(0, 0, 0, .07); } - - .swagger-ui .model-box { background: rgba(0, 0, 0, .1); } - - .swagger-ui .prop-type { color: #aaaad4; } - - .swagger-ui table thead tr td, .swagger-ui table thead tr th { - border-bottom: 1px solid rgba(58, 64, 80, .2); - color: #b5bac9; - } - - .swagger-ui .parameter__name.required::after { color: rgba(230, 153, 153, .6); } - - .swagger-ui .topbar .download-url-wrapper .select-label { color: #f0f0f0; } - - .swagger-ui .topbar .download-url-wrapper .download-url-button { - background: #63a040; - color: #fff; - } - - .swagger-ui .info .title small { background: #7c8492; } - - .swagger-ui .info .title small.version-stamp { background-color: #7a9b27; } - - .swagger-ui .auth-container .errors { - background-color: #350d0d; - color: #b5bac9; - } - - .swagger-ui .errors-wrapper { - background: rgba(200, 50, 50, .1); - border: 2px solid #c83232; - } - - .swagger-ui .markdown code, .swagger-ui .renderedmarkdown code { - background: rgba(0, 0, 0, .05); - color: #c299e6; - } - - .swagger-ui .model-toggle:after { background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50cy8xLjEvIgogICB4bWxuczpjYz0iaHR0cDovL2NyZWF0aXZlY29tbW9ucy5vcmcvbnMjIgogICB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFREL3NvZGlwb2RpLTAuZHRkIgogICB4bWxuczppbmtzY2FwZT0iaHR0cDovL3d3dy5pbmtzY2FwZS5vcmcvbmFtZXNwYWNlcy9pbmtzY2FwZSIKICAgaW5rc2NhcGU6dmVyc2lvbj0iMS4wICg0MDM1YTRmYjQ5LCAyMDIwLTA1LTAxKSIKICAgc29kaXBvZGk6ZG9jbmFtZT0iZG93bmxvYWQyLnN2ZyIKICAgaWQ9InN2ZzQiCiAgIHZlcnNpb249IjEuMSIKICAgaGVpZ2h0PSIyNCIKICAgd2lkdGg9IjI0Ij4KICA8bWV0YWRhdGEKICAgICBpZD0ibWV0YWRhdGExMCI+CiAgICA8cmRmOlJERj4KICAgICAgPGNjOldvcmsKICAgICAgICAgcmRmOmFib3V0PSIiPgogICAgICAgIDxkYzpmb3JtYXQ+aW1hZ2Uvc3ZnK3htbDwvZGM6Zm9ybWF0PgogICAgICAgIDxkYzp0eXBlCiAgICAgICAgICAgcmRmOnJlc291cmNlPSJodHRwOi8vcHVybC5vcmcvZGMvZGNtaXR5cGUvU3RpbGxJbWFnZSIgLz4KICAgICAgPC9jYzpXb3JrPgogICAgPC9yZGY6UkRGPgogIDwvbWV0YWRhdGE+CiAgPGRlZnMKICAgICBpZD0iZGVmczgiIC8+CiAgPHNvZGlwb2RpOm5hbWVkdmlldwogICAgIGlua3NjYXBlOmN1cnJlbnQtbGF5ZXI9InN2ZzQiCiAgICAgaW5rc2NhcGU6d2luZG93LW1heGltaXplZD0iMSIKICAgICBpbmtzY2FwZTp3aW5kb3cteT0iLTkiCiAgICAgaW5rc2NhcGU6d2luZG93LXg9Ii05IgogICAgIGlua3NjYXBlOmN5PSIxMiIKICAgICBpbmtzY2FwZTpjeD0iMTIiCiAgICAgaW5rc2NhcGU6em9vbT0iMzQuNTgzMzMzIgogICAgIHNob3dncmlkPSJmYWxzZSIKICAgICBpZD0ibmFtZWR2aWV3NiIKICAgICBpbmtzY2FwZTp3aW5kb3ctaGVpZ2h0PSIxMDAxIgogICAgIGlua3NjYXBlOndpbmRvdy13aWR0aD0iMTkyMCIKICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIgogICAgIGlua3NjYXBlOnBhZ2VvcGFjaXR5PSIwIgogICAgIGd1aWRldG9sZXJhbmNlPSIxMCIKICAgICBncmlkdG9sZXJhbmNlPSIxMCIKICAgICBvYmplY3R0b2xlcmFuY2U9IjEwIgogICAgIGJvcmRlcm9wYWNpdHk9IjEiCiAgICAgYm9yZGVyY29sb3I9IiM2NjY2NjYiCiAgICAgcGFnZWNvbG9yPSIjZmZmZmZmIiAvPgogIDxwYXRoCiAgICAgc3R5bGU9ImZpbGw6I2ZmZmZmZiIKICAgICBpZD0icGF0aDIiCiAgICAgZD0iTTEwIDZMOC41OSA3LjQxIDEzLjE3IDEybC00LjU4IDQuNTlMMTAgMThsNi02eiIgLz4KPC9zdmc+Cg==) 50% no-repeat; } - - /* arrows for each operation and request are now white */ - .arrow, #large-arrow-up { fill: #fff; } - - #unlocked { fill: #fff; } - - ::-webkit-scrollbar-track { background-color: #646464 !important; } - - ::-webkit-scrollbar-thumb { - background-color: #242424 !important; - border: 2px solid #3e4346 !important; - } - - ::-webkit-scrollbar-button:vertical:start:decrement { - background: linear-gradient(130deg, #696969 40%, rgba(255, 0, 0, 0) 41%), linear-gradient(230deg, #696969 40%, transparent 41%), linear-gradient(0deg, #696969 40%, transparent 31%); - background-color: #b6b6b6; - } - - ::-webkit-scrollbar-button:vertical:end:increment { - background: linear-gradient(310deg, #696969 40%, transparent 41%), linear-gradient(50deg, #696969 40%, transparent 41%), linear-gradient(180deg, #696969 40%, transparent 31%); - background-color: #b6b6b6; - } - - ::-webkit-scrollbar-button:horizontal:end:increment { - background: linear-gradient(210deg, #696969 40%, transparent 41%), linear-gradient(330deg, #696969 40%, transparent 41%), linear-gradient(90deg, #696969 30%, transparent 31%); - background-color: #b6b6b6; - } - - ::-webkit-scrollbar-button:horizontal:start:decrement { - background: linear-gradient(30deg, #696969 40%, transparent 41%), linear-gradient(150deg, #696969 40%, transparent 41%), linear-gradient(270deg, #696969 30%, transparent 31%); - background-color: #b6b6b6; - } - - ::-webkit-scrollbar-button, ::-webkit-scrollbar-track-piece { background-color: #3e4346 !important; } - - .swagger-ui .black, .swagger-ui .checkbox, .swagger-ui .dark-gray, .swagger-ui .download-url-wrapper .loading, .swagger-ui .errors-wrapper .errors small, .swagger-ui .fallback, .swagger-ui .filter .loading, .swagger-ui .gray, .swagger-ui .hover-black:focus, .swagger-ui .hover-black:hover, .swagger-ui .hover-dark-gray:focus, .swagger-ui .hover-dark-gray:hover, .swagger-ui .hover-gray:focus, .swagger-ui .hover-gray:hover, .swagger-ui .hover-light-silver:focus, .swagger-ui .hover-light-silver:hover, .swagger-ui .hover-mid-gray:focus, .swagger-ui .hover-mid-gray:hover, .swagger-ui .hover-near-black:focus, .swagger-ui .hover-near-black:hover, .swagger-ui .hover-silver:focus, .swagger-ui .hover-silver:hover, .swagger-ui .light-silver, .swagger-ui .markdown pre, .swagger-ui .mid-gray, .swagger-ui .model .property, .swagger-ui .model .property.primitive, .swagger-ui .model-title, .swagger-ui .near-black, .swagger-ui .parameter__extension, .swagger-ui .parameter__in, .swagger-ui .prop-format, .swagger-ui .renderedmarkdown pre, .swagger-ui .response-col_links .response-undocumented, .swagger-ui .response-col_status .response-undocumented, .swagger-ui .silver, .swagger-ui section.models h4, .swagger-ui section.models h5, .swagger-ui span.token-not-formatted, .swagger-ui span.token-string, .swagger-ui table.headers .header-example, .swagger-ui table.model tr.description, .swagger-ui table.model tr.extension { color: #bfbfbf; } - - .swagger-ui .hover-white:focus, .swagger-ui .hover-white:hover, .swagger-ui .info .title small pre, .swagger-ui .topbar a, .swagger-ui .white { color: #fff; } - - .swagger-ui .bg-black-10, .swagger-ui .hover-bg-black-10:focus, .swagger-ui .hover-bg-black-10:hover, .swagger-ui .stripe-dark:nth-child(2n + 1) { background-color: rgba(0, 0, 0, .1); } - - .swagger-ui .bg-white-10, .swagger-ui .hover-bg-white-10:focus, .swagger-ui .hover-bg-white-10:hover, .swagger-ui .stripe-light:nth-child(2n + 1) { background-color: rgba(28, 28, 33, .1); } - - .swagger-ui .bg-light-silver, .swagger-ui .hover-bg-light-silver:focus, .swagger-ui .hover-bg-light-silver:hover, .swagger-ui .striped--light-silver:nth-child(2n + 1) { background-color: #6e6e6e; } - - .swagger-ui .bg-moon-gray, .swagger-ui .hover-bg-moon-gray:focus, .swagger-ui .hover-bg-moon-gray:hover, .swagger-ui .striped--moon-gray:nth-child(2n + 1) { background-color: #4d4d4d; } - - .swagger-ui .bg-light-gray, .swagger-ui .hover-bg-light-gray:focus, .swagger-ui .hover-bg-light-gray:hover, .swagger-ui .striped--light-gray:nth-child(2n + 1) { background-color: #2b2b2b; } - - .swagger-ui .bg-near-white, .swagger-ui .hover-bg-near-white:focus, .swagger-ui .hover-bg-near-white:hover, .swagger-ui .striped--near-white:nth-child(2n + 1) { background-color: #242424; } - - .swagger-ui .opblock-tag:hover, .swagger-ui section.models h4:hover { background: rgba(0, 0, 0, .02); } - - .swagger-ui .checkbox p, .swagger-ui .dialog-ux .modal-ux-content h4, .swagger-ui .dialog-ux .modal-ux-content p, .swagger-ui .dialog-ux .modal-ux-header h3, .swagger-ui .errors-wrapper .errors h4, .swagger-ui .errors-wrapper hgroup h4, .swagger-ui .info .base-url, .swagger-ui .info .title, .swagger-ui .info h1, .swagger-ui .info h2, .swagger-ui .info h3, .swagger-ui .info h4, .swagger-ui .info h5, .swagger-ui .info li, .swagger-ui .info p, .swagger-ui .info table, .swagger-ui .loading-container .loading::after, .swagger-ui .model, .swagger-ui .opblock .opblock-section-header h4, .swagger-ui .opblock .opblock-section-header > label, .swagger-ui .opblock .opblock-summary-description, .swagger-ui .opblock .opblock-summary-operation-id, .swagger-ui .opblock .opblock-summary-path, .swagger-ui .opblock .opblock-summary-path__deprecated, .swagger-ui .opblock-description-wrapper, .swagger-ui .opblock-description-wrapper h4, .swagger-ui .opblock-description-wrapper p, .swagger-ui .opblock-external-docs-wrapper, .swagger-ui .opblock-external-docs-wrapper h4, .swagger-ui .opblock-external-docs-wrapper p, .swagger-ui .opblock-tag small, .swagger-ui .opblock-title_normal, .swagger-ui .opblock-title_normal h4, .swagger-ui .opblock-title_normal p, .swagger-ui .parameter__name, .swagger-ui .parameter__type, .swagger-ui .response-col_links, .swagger-ui .response-col_status, .swagger-ui .responses-inner h4, .swagger-ui .responses-inner h5, .swagger-ui .scheme-container .schemes > label, .swagger-ui .scopes h2, .swagger-ui .servers > label, .swagger-ui .tab li, .swagger-ui label, .swagger-ui select, .swagger-ui table.headers td { color: #b5bac9; } - - .swagger-ui .download-url-wrapper .failed, .swagger-ui .filter .failed, .swagger-ui .model-deprecated-warning, .swagger-ui .parameter__deprecated, .swagger-ui .parameter__name.required span, .swagger-ui table.model tr.property-row .star { color: #e69999; } - - .swagger-ui .opblock-body pre.microlight, .swagger-ui textarea.curl { - background: #41444e; - border-radius: 4px; - color: #fff; - } - - .swagger-ui .expand-methods svg, .swagger-ui .expand-methods:hover svg { fill: #bfbfbf; } - - .swagger-ui .auth-container, .swagger-ui .dialog-ux .modal-ux-header { border-bottom: 1px solid #2e2e2e; } - - .swagger-ui .topbar .download-url-wrapper .select-label select, .swagger-ui .topbar .download-url-wrapper input[type=text] { border: 2px solid #63a040; } - - .swagger-ui .info a, .swagger-ui .info a:hover, .swagger-ui .scopes h2 a { color: #99bde6; } - - /* Dark Scrollbar */ - ::-webkit-scrollbar { - width: 14px; - height: 14px; - } - - ::-webkit-scrollbar-button { - background-color: #3e4346 !important; - } - - ::-webkit-scrollbar-track { - background-color: #646464 !important; - } - - ::-webkit-scrollbar-track-piece { - background-color: #3e4346 !important; - } - - ::-webkit-scrollbar-thumb { - height: 50px; - background-color: #242424 !important; - border: 2px solid #3e4346 !important; - } - - ::-webkit-scrollbar-corner {} - - ::-webkit-resizer {} - - ::-webkit-scrollbar-button:vertical:start:decrement { - background: - linear-gradient(130deg, #696969 40%, rgba(255, 0, 0, 0) 41%), - linear-gradient(230deg, #696969 40%, rgba(0, 0, 0, 0) 41%), - linear-gradient(0deg, #696969 40%, rgba(0, 0, 0, 0) 31%); - background-color: #b6b6b6; - } - - ::-webkit-scrollbar-button:vertical:end:increment { - background: - linear-gradient(310deg, #696969 40%, rgba(0, 0, 0, 0) 41%), - linear-gradient(50deg, #696969 40%, rgba(0, 0, 0, 0) 41%), - linear-gradient(180deg, #696969 40%, rgba(0, 0, 0, 0) 31%); - background-color: #b6b6b6; - } - - ::-webkit-scrollbar-button:horizontal:end:increment { - background: - linear-gradient(210deg, #696969 40%, rgba(0, 0, 0, 0) 41%), - linear-gradient(330deg, #696969 40%, rgba(0, 0, 0, 0) 41%), - linear-gradient(90deg, #696969 30%, rgba(0, 0, 0, 0) 31%); - background-color: #b6b6b6; - } - - ::-webkit-scrollbar-button:horizontal:start:decrement { - background: - linear-gradient(30deg, #696969 40%, rgba(0, 0, 0, 0) 41%), - linear-gradient(150deg, #696969 40%, rgba(0, 0, 0, 0) 41%), - linear-gradient(270deg, #696969 30%, rgba(0, 0, 0, 0) 31%); - background-color: #b6b6b6; - } +.swagger-ui .json-schema-2020-12-keyword { + margin: 5px 0 +} + +.swagger-ui .json-schema-2020-12-keyword__children { + margin: 0 0 0 20px; + padding: 0 +} + +.swagger-ui .json-schema-2020-12-keyword__children--collapsed { + display: none +} + +.swagger-ui .json-schema-2020-12-keyword__name { + font-size: 12px; + font-weight: 700; + margin-left: 20px +} + +.swagger-ui .json-schema-2020-12-keyword__name--primary { + color: #3b4151; + font-style: normal +} + +.swagger-ui .json-schema-2020-12-keyword__name--secondary { + color: #6b6b6b; + font-style: italic +} + +.swagger-ui .json-schema-2020-12-keyword__name--extension { + color: #929292; + font-style: italic +} + +.swagger-ui .json-schema-2020-12-keyword__value { + color: #6b6b6b; + font-size: 12px; + font-style: italic; + font-weight: 400 +} + +.swagger-ui .json-schema-2020-12-keyword__value--primary { + color: #3b4151; + font-style: normal +} + +.swagger-ui .json-schema-2020-12-keyword__value--secondary { + color: #6b6b6b; + font-style: italic +} + +.swagger-ui .json-schema-2020-12-keyword__value--extension { + color: #929292; + font-style: italic +} + +.swagger-ui .json-schema-2020-12-keyword__value--warning { + color: #3b4151; + color: red; + display: inline-block; + font-family: monospace; + font-style: normal; + font-weight: 600; + line-height: 1.5; + margin-left: 10px; + padding: 1px 4px +} + +.swagger-ui .json-schema-2020-12-keyword__name--secondary+.json-schema-2020-12-keyword__value--secondary:before { + content: "=" +} + +.swagger-ui .json-schema-2020-12__attribute { + color: #3b4151; + font-family: monospace; + font-size: 12px; + padding-left: 10px; + text-transform: lowercase +} + +.swagger-ui .json-schema-2020-12__attribute--primary { + color: #55a +} + +.swagger-ui .json-schema-2020-12__attribute--muted { + color: gray +} + +.swagger-ui .json-schema-2020-12__attribute--warning { + color: red +} + +.swagger-ui .json-schema-2020-12-json-viewer { + margin: 5px 0 +} + +.swagger-ui .json-schema-2020-12-json-viewer__children { + margin: 0 0 0 20px; + padding: 0 +} + +.swagger-ui .json-schema-2020-12-json-viewer__children--collapsed { + display: none +} + +.swagger-ui .json-schema-2020-12-json-viewer__name { + font-size: 12px; + font-weight: 700; + margin-left: 20px +} + +.swagger-ui .json-schema-2020-12-json-viewer__name--primary { + color: #3b4151; + font-style: normal +} + +.swagger-ui .json-schema-2020-12-json-viewer__name--secondary { + color: #6b6b6b; + font-style: italic +} + +.swagger-ui .json-schema-2020-12-json-viewer__name--extension { + color: #929292; + font-style: italic +} + +.swagger-ui .json-schema-2020-12-json-viewer__value { + color: #6b6b6b; + font-size: 12px; + font-style: italic; + font-weight: 400 +} + +.swagger-ui .json-schema-2020-12-json-viewer__value--primary { + color: #3b4151; + font-style: normal +} + +.swagger-ui .json-schema-2020-12-json-viewer__value--secondary { + color: #6b6b6b; + font-style: italic +} + +.swagger-ui .json-schema-2020-12-json-viewer__value--extension { + color: #929292; + font-style: italic +} + +.swagger-ui .json-schema-2020-12-json-viewer__value--warning { + color: #3b4151; + color: red; + display: inline-block; + font-family: monospace; + font-style: normal; + font-weight: 600; + line-height: 1.5; + margin-left: 10px; + padding: 1px 4px +} + +.swagger-ui .json-schema-2020-12-json-viewer__name--secondary+.json-schema-2020-12-json-viewer__value--secondary:before { + content: "=" +} + +.swagger-ui .json-schema-2020-12 { + background-color: rgba(0,0,0,.05); + padding: 12px 0 12px 20px +} + +.swagger-ui .json-schema-2020-12--embedded { + background-color: inherit; + padding-bottom: 0; + padding-left: inherit; + padding-right: inherit; + padding-top: 0 +} + +.swagger-ui .json-schema-2020-12-body { + margin: 2px 0 +} + +.swagger-ui .json-schema-2020-12-body--collapsed { + display: none +} + +.swagger-ui .json-schema-2020-12-accordion { + outline: none; + padding-left: 0; + border: none; + background-color: transparent; +} + +.swagger-ui .json-schema-2020-12-accordion__children { + display: inline-block +} + +.swagger-ui .json-schema-2020-12-accordion__icon { + display: inline-block; + height: 18px; + vertical-align: bottom; + width: 18px +} + +.swagger-ui .json-schema-2020-12-accordion__icon--expanded { + transform: rotate(-90deg); + transform-origin: 50% 50%; + transition: transform .15s ease-in +} + +.swagger-ui .json-schema-2020-12-accordion__icon--collapsed { + transform: rotate(0deg); + transform-origin: 50% 50%; + transition: transform .15s ease-in +} + +.swagger-ui .json-schema-2020-12-accordion__icon svg { + height: 20px; + width: 20px +} + +.swagger-ui .json-schema-2020-12-expand-deep-button { + color: #505050; + color: #55a; + font-family: sans-serif; + font-size: 12px; + padding-right: 0; + border: none; + background-color: transparent; +} + +.swagger-ui .model-box .json-schema-2020-12:not(.json-schema-2020-12--embedded)>.json-schema-2020-12-head .json-schema-2020-12__title:first-of-type { + font-size: 16px +} + +.swagger-ui .model-box>.json-schema-2020-12 { + margin: 0 +} + +.swagger-ui .model-box .json-schema-2020-12 { + background-color: transparent; + padding: 0 +} + +.swagger-ui .model-box .json-schema-2020-12-accordion,.swagger-ui .model-box .json-schema-2020-12-expand-deep-button { + background-color: transparent +} + +.swagger-ui .models .json-schema-2020-12:not(.json-schema-2020-12--embedded)>.json-schema-2020-12-head .json-schema-2020-12__title:first-of-type { + font-size: 16px +} + +.swagger-ui .models .json-schema-2020-12:not(.json-schema-2020-12--embedded) { + overflow-x: auto; + width: calc(100% - 40px) +} + +.response-col_description__inner { + font-weight: 500; + font-size: 1.2em; } diff --git a/installer.py b/installer.py index 8609ff953..835478690 100644 --- a/installer.py +++ b/installer.py @@ -56,6 +56,7 @@ args = Dot({ }) git_commit = "unknown" diffusers_commit = "unknown" +restart_required = False extensions_commit = { # force specific commit for extensions 'sd-webui-controlnet': 'ecd33eb', 'adetailer': 'a89c01d' @@ -345,6 +346,8 @@ def installed(package, friendly: str = None, reload = False, quiet = False): # p log.warning(f'Install: package="{p[0]}" installed={pkg_version} required={p[1]} allowing experimental') else: log.warning(f'Install: package="{p[0]}" installed={pkg_version} required={p[1]} version mismatch') + global restart_required # pylint: disable=global-statement + restart_required = True ok = ok and (exact or args.experimental) else: if not quiet: @@ -605,7 +608,9 @@ def check_diffusers(): if args.skip_git: install('diffusers') return - sha = '5e181eddfe7e44c1444a2511b0d8e21d177850a0' # diffusers commit hash + sha = 'af769881d37fe916afef2c47279f66c79f5f2714' # diffusers commit hash + # if args.use_rocm or args.use_zluda or args.use_directml: + # sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else -1) cur = opts.get('diffusers_version', '') if minor > -1 else '' @@ -626,19 +631,19 @@ def check_transformers(): t_start = time.time() if args.skip_all or args.skip_git or args.experimental: return - pkg_transofmers = pkg_resources.working_set.by_key.get('transformers', None) + pkg_transformers = pkg_resources.working_set.by_key.get('transformers', None) pkg_tokenizers = pkg_resources.working_set.by_key.get('tokenizers', None) if args.use_directml: target_transformers = '4.52.4' target_tokenizers = '0.21.4' else: - target_transformers = '4.56.1' - target_tokenizers = '0.22.0' - if (pkg_transofmers is None) or ((pkg_transofmers.version != target_transformers) or (pkg_tokenizers is None) or ((pkg_tokenizers.version != target_tokenizers) and (not args.experimental))): - if pkg_transofmers is None: + target_transformers = '4.57.1' + target_tokenizers = '0.22.1' + if (pkg_transformers is None) or ((pkg_transformers.version != target_transformers) or (pkg_tokenizers is None) or ((pkg_tokenizers.version != target_tokenizers) and (not args.experimental))): + if pkg_transformers is None: log.info(f'Transformers install: version={target_transformers}') else: - log.info(f'Transformers update: current={pkg_transofmers.version} target={target_transformers}') + log.info(f'Transformers update: current={pkg_transformers.version} target={target_transformers}') pip('uninstall --yes transformers', ignore=True, quiet=True, uv=False) pip(f'install --upgrade tokenizers=={target_tokenizers}', ignore=False, quiet=True, uv=False) pip(f'install --upgrade transformers=={target_transformers}', ignore=False, quiet=True, uv=False) @@ -674,82 +679,83 @@ def install_rocm_zluda(): if args.skip_all or args.skip_requirements: return torch_command from modules import rocm - if not rocm.is_installed: - log.warning('ROCm: could not find ROCm toolkit installed') - log.info('Using CPU-only torch') - return os.environ.get('TORCH_COMMAND', 'torch torchvision') - log.info('ROCm: AMD toolkit detected') - # if not is_windows: - # os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow-rocm') - - device = None + amd_gpus = [] try: - amd_gpus = rocm.get_agents() - if len(amd_gpus) == 0: - log.warning('ROCm: no agent was found') + if sys.platform == "win32" and not rocm.is_installed: + amd_gpus = rocm.driver_get_agents() else: - log.info(f'ROCm: agents={[gpu.name for gpu in amd_gpus]}') - if args.device_id is None: - index = 0 - for idx, gpu in enumerate(amd_gpus): - index = idx - # if gpu.name.startswith('gfx11') and os.environ.get('TENSORFLOW_PACKAGE') == 'tensorflow-rocm': # do not use tensorflow-rocm for navi 3x - # os.environ['TENSORFLOW_PACKAGE'] = 'tensorflow==2.13.0' - if not gpu.is_apu: - # although apu was found, there can be a dedicated card. do not break loop. - # if no dedicated card was found, apu will be used. - break - os.environ.setdefault('HIP_VISIBLE_DEVICES', str(index)) - device = amd_gpus[index] - else: - device_id = int(args.device_id) - if device_id < len(amd_gpus): - device = amd_gpus[device_id] + amd_gpus = rocm.get_agents() + log.info('ROCm: AMD toolkit detected') except Exception as e: log.warning(f'ROCm agent enumerator failed: {e}') + #os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow') + + device = None + if len(amd_gpus) == 0: + log.warning('ROCm: no agent was found') + else: + log.info(f'ROCm: agents={[gpu.name for gpu in amd_gpus]}') + if args.device_id is None: + index = 0 + for idx, gpu in enumerate(amd_gpus): + index = idx + if not gpu.is_apu: + # although apu was found, there can be a dedicated card. do not break loop. + # if no dedicated card was found, apu will be used. + break + os.environ.setdefault('HIP_VISIBLE_DEVICES', str(index)) + device = amd_gpus[index] + else: + device_id = int(args.device_id) + if device_id < len(amd_gpus): + device = amd_gpus[device_id] + + if sys.platform == "win32" and args.use_rocm and not rocm.is_installed: + check_python(supported_minors=[11, 12, 13], reason='ROCm backend requires a Python version between 3.11 and 3.13') + install(f"rocm rocm-sdk-core --index-url https://rocm.nightlies.amd.com/v2-staging/{device.therock}") + rocm.refresh() + msg = f'ROCm: version={rocm.version}' if device is not None: msg += f', using agent {device.name}' log.info(msg) - if sys.platform == "win32": # TODO install: enable ROCm for windows when available - #check_python(supported_minors=[10, 11, 12, 13], reason='ZLUDA backend requires a Python version between 3.10 and 3.13') + if sys.platform == "win32": + if args.use_rocm: # TODO install: switch to pytorch source when it becomes available + if device is not None and isinstance(rocm.environment, rocm.PythonPackageEnvironment): # TheRock + check_python(supported_minors=[11, 12, 13], reason='ROCm backend requires a Python version between 3.11 and 3.13') + torch_command = os.environ.get('TORCH_COMMAND', f'torch torchvision --index-url https://rocm.nightlies.amd.com/v2-staging/{device.therock}') + else: + check_python(supported_minors=[12], reason='ROCm Windows preview requires Python version 3.12') + torch_command = os.environ.get('TORCH_COMMAND', '--no-cache-dir https://repo.radeon.com/rocm/windows/rocm-rel-6.4.4/torch-2.8.0a0%2Bgitfc14c65-cp312-cp312-win_amd64.whl https://repo.radeon.com/rocm/windows/rocm-rel-6.4.4/torchvision-0.24.0a0%2Bc85f008-cp312-cp312-win_amd64.whl') + else: + #check_python(supported_minors=[10, 11, 12, 13], reason='ZLUDA backend requires a Python version between 3.10 and 3.13') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1+cu118 torchvision==0.22.1+cu118 --index-url https://download.pytorch.org/whl/cu118') - if args.device_id is not None: - if os.environ.get('HIP_VISIBLE_DEVICES', None) is not None: - log.warning('Setting HIP_VISIBLE_DEVICES and --device-id at the same time may be mistake.') - os.environ['HIP_VISIBLE_DEVICES'] = args.device_id - del args.device_id + if args.device_id is not None: + if os.environ.get('HIP_VISIBLE_DEVICES', None) is not None: + log.warning('Setting HIP_VISIBLE_DEVICES and --device-id at the same time may be mistake.') + os.environ['HIP_VISIBLE_DEVICES'] = args.device_id + del args.device_id - error = None - from modules import zluda_installer - try: - if args.reinstall or zluda_installer.is_reinstall_needed(): - zluda_installer.uninstall() - zluda_installer.install() - zluda_installer.set_default_agent(device) - except Exception as e: - error = e - log.warning(f'Failed to install ZLUDA: {e}') + from modules import zluda_installer + try: + if args.reinstall or zluda_installer.is_reinstall_needed(): + zluda_installer.uninstall() + zluda_installer.install() + zluda_installer.set_default_agent(device) + except Exception as e: + log.warning(f'Failed to install ZLUDA: {e}') - if error is None: try: zluda_installer.load() - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1+cu118 torchvision==0.22.1+cu118 --index-url https://download.pytorch.org/whl/cu118') except Exception as e: - error = e log.warning(f'Failed to load ZLUDA: {e}') - if error is not None: - log.info('Using CPU-only torch') - torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') else: #check_python(supported_minors=[10, 11, 12, 13], reason='ROCm backend requires a Python version between 3.10 and 3.13') - if os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", None) is None: - os.environ.setdefault('TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL', '1') - if args.use_nightly: if rocm.version is None or float(rocm.version) >= 6.4: # assume the latest if version check fails torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm6.4') @@ -779,7 +785,7 @@ def install_rocm_zluda(): log.info(f'ROCm: HSA_OVERRIDE_GFX_VERSION auto config skipped: device={device.name if device is not None else None} version={os.environ.get("HSA_OVERRIDE_GFX_VERSION", None)}') else: gfx_ver = device.get_gfx_version() - if gfx_ver is not None: + if gfx_ver is not None and device.name.removeprefix("gfx") != gfx_ver.replace(".", ""): os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', gfx_ver) log.info(f'ROCm: HSA_OVERRIDE_GFX_VERSION config overridden: device={device.name} version={os.environ.get("HSA_OVERRIDE_GFX_VERSION", None)}') @@ -793,27 +799,6 @@ def install_ipex(): args.use_ipex = True # pylint: disable=attribute-defined-outside-init log.info('IPEX: Intel OneAPI toolkit detected') - if os.environ.get("NEOReadDebugKeys", None) is None: - os.environ.setdefault('NEOReadDebugKeys', '1') - - if os.environ.get("ClDeviceGlobalMemSizeAvailablePercent", None) is None: - os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') - - if os.environ.get("SYCL_CACHE_PERSISTENT", None) is None: - os.environ.setdefault('SYCL_CACHE_PERSISTENT', '1') # Jit cache - - if os.environ.get("PYTORCH_ENABLE_XPU_FALLBACK", None) is None: - os.environ.setdefault('PYTORCH_ENABLE_XPU_FALLBACK', '1') # CPU fallback for unsupported ops - - if os.environ.get("UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS", None) is None: - os.environ.setdefault('UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS', '1') # Work around the 4G alloc limit on Alchemist - - # FP64 emulation causes random UR Errors - #if os.environ.get("OverrideDefaultFP64Settings", None) is None: - # os.environ.setdefault('OverrideDefaultFP64Settings', '1') - #if os.environ.get("IGC_EnableDPEmulation", None) is None: - # os.environ.setdefault('IGC_EnableDPEmulation', '1') # FP64 Emulation - if args.use_nightly: torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/xpu') else: @@ -827,20 +812,17 @@ def install_ipex(): def install_openvino(): t_start = time.time() log.info('OpenVINO: selected') - #check_python(supported_minors=[10, 11, 12, 13], reason='OpenVINO backend requires a Python version between 3.10 and 3.13') + os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX') + #check_python(supported_minors=[10, 11, 12, 13], reason='OpenVINO backend requires a Python version between 3.10 and 3.13') if sys.platform == 'darwin': torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.8.0 torchvision==0.23.0') else: torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.8.0+cpu torchvision==0.23.0 --index-url https://download.pytorch.org/whl/cpu') - install(os.environ.get('OPENVINO_COMMAND', 'openvino==2025.3.0'), 'openvino') - install(os.environ.get('NNCF_COMMAND', 'nncf==2.18.0'), 'nncf') - os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX') - if os.environ.get("NEOReadDebugKeys", None) is None: - os.environ.setdefault('NEOReadDebugKeys', '1') - if os.environ.get("ClDeviceGlobalMemSizeAvailablePercent", None) is None: - os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') + if not (args.skip_all or args.skip_requirements): + install(os.environ.get('OPENVINO_COMMAND', 'openvino==2025.3.0'), 'openvino') + install(os.environ.get('NNCF_COMMAND', 'nncf==2.18.0'), 'nncf') ts('openvino', t_start) return torch_command @@ -874,6 +856,7 @@ def install_torch_addons(): install('pillow-jxl-plugin==1.3.4', 'pillow-jxl-plugin') if not args.experimental: uninstall('wandb', quiet=True) + uninstall('pynvml', quiet=True) ts('addons', t_start) @@ -921,9 +904,9 @@ def check_torch(): if torch_command != '': pass else: - is_cuda_available = allow_cuda and (shutil.which('nvidia-smi') is not None or args.use_xformers or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))) - is_rocm_available = allow_rocm and rocm.is_installed - is_ipex_available = allow_ipex and (args.use_ipex or shutil.which('sycl-ls') is not None or shutil.which('sycl-ls.exe') is not None or os.environ.get('ONEAPI_ROOT') is not None or os.path.exists('/opt/intel/oneapi') or os.path.exists("C:/Program Files (x86)/Intel/oneAPI") or os.path.exists("C:/oneAPI")) + is_cuda_available = allow_cuda and (args.use_cuda or shutil.which('nvidia-smi') is not None or args.use_xformers or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))) + is_rocm_available = allow_rocm and (args.use_rocm or args.use_zluda or rocm.is_installed) + is_ipex_available = allow_ipex and (args.use_ipex or shutil.which('sycl-ls') is not None or shutil.which('sycl-ls.exe') is not None or os.environ.get('ONEAPI_ROOT') is not None or os.path.exists('/opt/intel/oneapi') or os.path.exists("C:/Program Files (x86)/Intel/oneAPI") or os.path.exists("C:/oneAPI") or os.path.exists("C:/Program Files/Intel/Intel Graphics Software")) if is_cuda_available and args.use_cuda: # prioritize cuda torch_command = install_cuda() @@ -950,63 +933,69 @@ def check_torch(): install(torch_command, 'torch torchvision') install('onnxruntime-directml', 'onnxruntime-directml', ignore=True) else: - if args.use_zluda: - log.warning("ZLUDA failed to initialize: no HIP SDK found") log.warning('Torch: CPU-only version installed') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') - if 'torch' in torch_command and not args.version: + if args.version: + return + + if 'torch' in torch_command: if not installed('torch'): log.info(f'Torch: download and install in progress... cmd="{torch_command}"') install('--upgrade pip', 'pip', reinstall=True) # pytorch rocm is too large for older pip install(torch_command, 'torch torchvision', quiet=True) - else: + + try: + import torch try: - import torch - log.info(f'Torch {torch.__version__}') - if args.use_ipex and allow_ipex: - try: - import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import - log.info(f'Torch backend: Intel IPEX {ipex.__version__}') - except Exception: - log.warning('IPEX: not found') - if shutil.which('icpx') is not None: - log.info(f'{os.popen("icpx --version").read().rstrip()}') - for device in range(torch.xpu.device_count()): - log.info(f'Torch detected GPU: {torch.xpu.get_device_name(device)} VRAM {round(torch.xpu.get_device_properties(device).total_memory / 1024 / 1024)} Compute Units {torch.xpu.get_device_properties(device).max_compute_units}') - elif torch.cuda.is_available() and (allow_cuda or allow_rocm): - # log.debug(f'Torch allocator: {torch.cuda.get_allocator_backend()}') - if torch.version.cuda and allow_cuda: - log.info(f'Torch backend: nVidia CUDA {torch.version.cuda} cuDNN {torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else "N/A"}') - elif torch.version.hip and allow_rocm: - log.info(f'Torch backend: AMD ROCm HIP {torch.version.hip}') + import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import + log.info(f'Torch backend: type=IPEX version={ipex.__version__}') + except Exception: + pass + if 'cpu' in torch.__version__: + if is_cuda_available: + if args.use_cuda: + log.warning(f'Torch: version="{torch.__version__}" CPU version installed and CUDA is selected - reinstalling') + install(torch_command, 'torch torchvision', quiet=True, reinstall=True, force=True) # foce reinstall else: - log.warning('Unknown Torch backend') - for device in [torch.cuda.device(i) for i in range(torch.cuda.device_count())]: - log.info(f'Torch detected GPU: {torch.cuda.get_device_name(device)} VRAM {round(torch.cuda.get_device_properties(device).total_memory / 1024 / 1024)} Arch {torch.cuda.get_device_capability(device)} Cores {torch.cuda.get_device_properties(device).multi_processor_count}') + log.warning(f'Torch: version="{torch.__version__}" CPU version installed and CUDA is available - consider reinstalling') + elif is_rocm_available: + if args.use_rocm: + log.warning(f'Torch: version="{torch.__version__}" CPU version installed and ROCm is selected - reinstalling') + install(torch_command, 'torch torchvision', quiet=True, reinstall=True, force=True) # foce reinstall + else: + log.warning(f'Torch: version="{torch.__version__}" CPU version installed and ROCm is available - consider reinstalling') + if hasattr(torch, "xpu") and torch.xpu.is_available() and allow_ipex: + if shutil.which('icpx') is not None: + log.info(f'{os.popen("icpx --version").read().rstrip()}') + for device in range(torch.xpu.device_count()): + log.info(f'Torch detected: gpu="{torch.xpu.get_device_name(device)}" vram={round(torch.xpu.get_device_properties(device).total_memory / 1024 / 1024)} units={torch.xpu.get_device_properties(device).max_compute_units}') + elif torch.cuda.is_available() and (allow_cuda or allow_rocm): + if torch.version.cuda and allow_cuda: + log.info(f'Torch backend: version="{torch.__version__}" type=CUDA CUDA={torch.version.cuda} cuDNN={torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else "N/A"}') + elif torch.version.hip and allow_rocm: + log.info(f'Torch backend: version="{torch.__version__}" type=ROCm HIP={torch.version.hip}') else: - try: - if args.use_directml and allow_directml: - import torch_directml # pylint: disable=import-error - dml_ver = pkg_resources.get_distribution("torch-directml") - log.info(f'Torch backend: DirectML ({dml_ver})') - for i in range(0, torch_directml.device_count()): - log.info(f'Torch detected GPU: {torch_directml.device_name(i)}') - except Exception: - log.warning("Torch reports CUDA not available") - except Exception as e: - log.error(f'Torch cannot load: {e}') - if not args.ignore: - sys.exit(1) - if rocm.is_installed: - if sys.platform == "win32": # CPU, DirectML, ZLUDA - rocm.conceal() - elif rocm.is_wsl: # WSL ROCm + log.warning('Unknown Torch backend') + for device in [torch.cuda.device(i) for i in range(torch.cuda.device_count())]: + log.info(f'Torch detected: gpu="{torch.cuda.get_device_name(device)}" vram={round(torch.cuda.get_device_properties(device).total_memory / 1024 / 1024)} arch={torch.cuda.get_device_capability(device)} cores={torch.cuda.get_device_properties(device).multi_processor_count}') + else: try: - rocm.load_hsa_runtime() - except OSError: - log.error("ROCm: failed to preload HSA runtime") - if args.version: - return + if args.use_directml and allow_directml: + import torch_directml # pylint: disable=import-error + dml_ver = pkg_resources.get_distribution("torch-directml") + log.warning(f'Torch backend: DirectML ({dml_ver})') + log.warning('DirectML: end-of-life') + for i in range(0, torch_directml.device_count()): + log.info(f'Torch detected GPU: {torch_directml.device_name(i)}') + except Exception: + log.warning("Torch reports CUDA not available") + except Exception as e: + log.error(f'Torch cannot load: {e}') + if not args.ignore: + sys.exit(1) + + if rocm.is_installed: + rocm.postinstall() if not args.skip_all: install_torch_addons() check_cudnn() @@ -1264,6 +1253,17 @@ def install_pydantic(): reload('pydantic', '1.10.21') +def install_insightface(): + install('git+https://github.com/deepinsight/insightface@29b6cd65aa0e9ae3b6602de3c52e9d8949c8ee86#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches + if args.new: + uninstall('albumentations') + install('albumentationsx') + else: + uninstall('albumentationsx') + install('albumentations==1.4.3', ignore=True, quiet=True) + install_pydantic() + + def install_optional(): t_start = time.time() log.info('Installing optional requirements...') @@ -1277,8 +1277,6 @@ def install_optional(): install('nvidia-ml-py', ignore=True, quiet=True) install('ultralytics==8.3.40', ignore=True, quiet=True) install('Cython', ignore=True, quiet=True) - install('git+https://github.com/deepinsight/insightface@554a05561cb71cfebb4e012dfea48807f845a0c2#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches - install('albumentations==1.4.3', ignore=True, quiet=True) install('av', ignore=True, quiet=True) install('gguf', ignore=True) try: @@ -1355,11 +1353,18 @@ def set_environment(): allocator += ',backend:cudaMallocAsync' if opts.get("torch_expandable_segments", False): allocator += ',expandable_segments:True' + os.environ.setdefault('PYTORCH_ALLOC_CONF', allocator) os.environ.setdefault('PYTORCH_CUDA_ALLOC_CONF', allocator) os.environ.setdefault('PYTORCH_HIP_ALLOC_CONF', allocator) log.debug(f'Torch allocator: "{allocator}"') - if sys.platform == 'darwin': - os.environ.setdefault('PYTORCH_ENABLE_MPS_FALLBACK', '1') + os.environ.setdefault('TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL', '1') + os.environ.setdefault('NEOReadDebugKeys', '1') + os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') + os.environ.setdefault('SYCL_CACHE_PERSISTENT', '1') + os.environ.setdefault('UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS', '1') + os.environ.setdefault('PYTORCH_ENABLE_XPU_FALLBACK', '1') + os.environ.setdefault('PYTORCH_ENABLE_MPS_FALLBACK', '1') + os.environ.setdefault('TOKENIZERS_PARALLELISM', '0') def check_extensions(): @@ -1634,7 +1639,6 @@ def add_args(parser): group_http.add_argument("--cors-regex", type=str, default=os.environ.get("SD_CORSREGEX", None), help="Allowed CORS origins as regular expression, default: %(default)s") group_http.add_argument('--subpath', type=str, default=os.environ.get("SD_SUBPATH", None), help='Customize the URL subpath for usage with reverse proxy') group_http.add_argument("--autolaunch", default=os.environ.get("SD_AUTOLAUNCH", False), action='store_true', help="Open the UI URL in the system's default browser upon launch") - group_http.add_argument('--docs', default=os.environ.get("SD_DOCS", False), action='store_true', help = "Mount API docs, default: %(default)s") group_http.add_argument("--auth", type=str, default=os.environ.get("SD_AUTH", None), help='Set access authentication like "user:pwd,user:pwd""') group_http.add_argument("--auth-file", type=str, default=os.environ.get("SD_AUTHFILE", None), help='Set access authentication using file, default: %(default)s') group_http.add_argument("--allowed-paths", nargs='+', default=[], type=str, required=False, help="add additional paths to paths allowed for web access") @@ -1655,11 +1659,11 @@ def add_args(parser): group_log = parser.add_argument_group('Logging') group_log.add_argument("--log", type=str, default=os.environ.get("SD_LOG", None), help="Set log file, default: %(default)s") - group_log.add_argument('--debug', default=os.environ.get("SD_DEBUG",False), action='store_true', help="Run with debug logging, default: %(default)s") + group_log.add_argument('--debug', default=not os.environ.get("SD_NODEBUG",False), action='store_true', help="Run with debug logging, default: %(default)s") group_log.add_argument("--trace", default=os.environ.get("SD_TRACE", False), action='store_true', help="Run with trace logging, default: %(default)s") group_log.add_argument("--profile", default=os.environ.get("SD_PROFILE", False), action='store_true', help="Run profiler, default: %(default)s") - group_log.add_argument('--docs', default=os.environ.get("SD_DOCS", False), action='store_true', help="Mount API docs, default: %(default)s") - group_log.add_argument("--api-log", default=os.environ.get("SD_APILOG", False), action='store_true', help="Log all API requests") + group_log.add_argument('--docs', default=not os.environ.get("SD_NODOCS", False), action='store_true', help = "Mount API docs, default: %(default)s") + group_log.add_argument("--api-log", default=not os.environ.get("SD_NOAPILOG", False), action='store_true', help="Log all API requests") group_nargs = parser.add_argument_group('Other') group_nargs.add_argument('args', type=str, nargs='*', help=argparse.SUPPRESS) diff --git a/javascript/control.js b/javascript/control.js index 1c1c8b3cc..e0bb550ba 100644 --- a/javascript/control.js +++ b/javascript/control.js @@ -2,13 +2,8 @@ function controlInputMode(inputMode, ...args) { const updateEl = gradioApp().getElementById('control_update'); if (updateEl) updateEl.click(); const tab = gradioApp().querySelector('#control-tab-input button.selected'); - if (!tab) return ['Select', ...args]; + if (!tab) return ['Image', ...args]; inputMode = tab.innerText; - if (inputMode === 'Image') { - if (!gradioApp().getElementById('control_input_select').classList.contains('hidden')) inputMode = 'Select'; - else if (!gradioApp().getElementById('control_input_resize').classList.contains('hidden')) inputMode = 'Outpaint'; - else if (!gradioApp().getElementById('control_input_inpaint').classList.contains('hidden')) inputMode = 'Inpaint'; - } return [inputMode, ...args]; } diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 3508ac572..95b5aca10 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -293,10 +293,11 @@ function extraNetworksSearchButton(event) { } function extraNetworksFilterVersion(event) { - // log('extraNetworksFilterVersion', event); + log('extraNetworksFilterVersion', event); const version = event.target.textContent.trim(); const activeTab = getENActiveTab(); - const cardContainer = gradioApp().querySelector(`#${activeTab}_model_cards`); + const activePage = getENActivePage().toLowerCase(); + const cardContainer = gradioApp().querySelector(`#${activeTab}_${activePage}_cards`); if (!cardContainer) return; if (cardContainer.dataset.activeVersion === version) { cardContainer.dataset.activeVersion = ''; diff --git a/javascript/guidance.js b/javascript/guidance.js new file mode 100644 index 000000000..11c2430d3 --- /dev/null +++ b/javascript/guidance.js @@ -0,0 +1,20 @@ +const guiders = { + None: '', + 'LSC: LayerSkipConfig': 'https://github.com/huggingface/diffusers/blob/041501aea92919c9c7f36e189fc9cf7d865ebb96/src/diffusers/hooks/layer_skip.py#L41', + 'CFG: ClassifierFreeGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.ClassifierFreeGuidance', + 'Auto: AutoGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.AutoGuidance', + 'Zero: ClassifierFreeZeroStar': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.ClassifierFreeZeroStarGuidance', + 'PAG: PerturbedAttentionGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.PerturbedAttentionGuidance', + 'APG: AdaptiveProjectedGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.AdaptiveProjectedGuidance', + 'SLG: SkipLayerGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.SkipLayerGuidance', + 'SEG: SmoothedEnergyGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.SmoothedEnergyGuidance', + 'TCFG: TangentialClassifierFreeGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.TangentialClassifierFreeGuidance', + 'FDG: FrequencyDecoupledGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.FrequencyDecoupledGuidance', +}; + +function getGuidanceDocs(guider) { + if (guider.label) guider = guider.label; + const url = guiders[guider]; + log('getGuidanceDocs', guider, url); + if (url) window.open(url, '_blank'); +} diff --git a/javascript/history.js b/javascript/history.js index ff2db4831..2a40b0859 100644 --- a/javascript/history.js +++ b/javascript/history.js @@ -45,7 +45,7 @@ function refreshHistory() { if (inferenceTypes.some((type) => entry.job.toLowerCase().startsWith(type))) entry.type = 'inference'; else if (ioTypes.some((type) => entry.job.toLowerCase().startsWith(type))) entry.type = 'io'; else entry.type = 'default'; - ts.push({ start, end: end.timestamp, label: entry.job, type: entry.type }); + if (start && end.timestamp) ts.push({ start, end: end.timestamp, label: entry.job, type: entry.type }); } } if (!ts.length) return; diff --git a/javascript/imageViewer.js b/javascript/imageViewer.js index 5631c173b..e7eaca7a8 100644 --- a/javascript/imageViewer.js +++ b/javascript/imageViewer.js @@ -15,11 +15,20 @@ function cycleImageFit() { log('cycleImageFit', current, next); } +function isInViewport(element) { + const rect = element.getBoundingClientRect(); + return rect.top >= 0 && rect.left >= 0 && rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) && rect.right <= (window.innerWidth || document.documentElement.clientWidth); +} + function closeModal(evt, force = false) { if (force) gradioApp().getElementById('lightboxModal').style.display = 'none'; if (previewDrag) return; if (evt?.button !== 0) return; gradioApp().getElementById('lightboxModal').style.display = 'none'; + let thumbnails = Array.from(gradioApp().querySelectorAll('.thumbnails .thumbnail-item')); + thumbnails = thumbnails.filter((el) => el.checkVisibility()); + if (thumbnails.length === 0) return; + thumbnails[0].focus(); } function modalImageSwitch(offset) { diff --git a/javascript/logMonitor.js b/javascript/logMonitor.js index 08f104b55..21c03e88d 100644 --- a/javascript/logMonitor.js +++ b/javascript/logMonitor.js @@ -78,6 +78,7 @@ async function logMonitor() { for (const line of lines) addLogLine(line); if (!logConnected) { logConnected = true; + monitorConnection(); xhrPost(`${window.api}/log`, { debug: 'connected' }); } } else { diff --git a/javascript/monitor.js b/javascript/monitor.js new file mode 100644 index 000000000..276e4bc0e --- /dev/null +++ b/javascript/monitor.js @@ -0,0 +1,35 @@ +async function updateIndicator(online, data, msg) { + const el = document.getElementById('logo_nav'); + if (!el || !data) return; + const status = online ? 'online' : 'offline'; + const date = new Date(); + const template = ` + Version: ${data.updated}
+ Commit: ${data.hash}
+ Branch: ${data.branch}
+ Status: ${status}
+ Since: ${date.toLocaleString()}
+ `; + if (online) { + el.dataset.hint = template; + el.style.backgroundColor = 'var(--sd-main-accent-color)'; + log('monitorConnection: online', data); + } else { + el.dataset.hint = template; + el.style.backgroundColor = 'var(--color-error)'; + log('monitorConnection: offline', msg); + } +} + +async function monitorConnection() { + try { + const res = await fetch(`${window.api}/version`); + const data = await res.json(); + const url = res.url.split('/sdapi')[0].replace('http', 'ws'); // update global url as ws need fqdn + const ws = new WebSocket(`${url}/queue/join`); + ws.onopen = () => updateIndicator(true, data, ''); + ws.onclose = () => updateIndicator(false, data, ''); + ws.onerror = (e) => updateIndicator(false, data, e.message); + ws.onmessage = (evt) => log('monitorConnection: message', evt.data); + } catch { /**/ } +} diff --git a/javascript/ui.js b/javascript/ui.js index a61297a54..8c5cd6630 100644 --- a/javascript/ui.js +++ b/javascript/ui.js @@ -601,4 +601,5 @@ async function reconnectUI() { const sd_model_observer = new MutationObserver(sd_model_callback); sd_model_observer.observe(sd_model, { attributes: true, childList: true, subtree: true }); log('reconnectUI'); + monitorConnection(); } diff --git a/launch.py b/launch.py index 8201b8275..40e1946fc 100755 --- a/launch.py +++ b/launch.py @@ -133,7 +133,7 @@ def run_pip(pkg, desc=None): # compatbility function desc = pkg for f in forbidden: if f in pkg: - debug_install('Blocked package installation: package={f}') + debug_install(f'Blocked package installation: package={f}') return True index_url_line = f' --index-url {index_url}' if index_url != '' else '' return run(f'"{sys.executable}" -m pip {pkg} --prefer-binary{index_url_line}', desc=f"Installing {desc}", errdesc=f"Couldn't install {desc}") @@ -297,6 +297,9 @@ def main(): installer.log.info(f'Installer time: {init_summary()}') get_custom_args() + if installer.restart_required: + installer.log.warning('Restart is recommended due to packages updates...') + uv, instance = start_server(immediate=True, server=None) t_server = time.time() t_monitor = time.time() diff --git a/models/Reference/OPPOer--Qwen-Image-Edit-2509-Pruning.jpg b/models/Reference/OPPOer--Qwen-Image-Edit-2509-Pruning.jpg new file mode 100644 index 000000000..eba34afbb Binary files /dev/null and b/models/Reference/OPPOer--Qwen-Image-Edit-2509-Pruning.jpg differ diff --git a/models/Reference/OPPOer--Qwen-Image-Edit-Pruning.jpg b/models/Reference/OPPOer--Qwen-Image-Edit-Pruning.jpg new file mode 100644 index 000000000..c960c4ca8 Binary files /dev/null and b/models/Reference/OPPOer--Qwen-Image-Edit-Pruning.jpg differ diff --git a/models/Reference/OPPOer--Qwen-Image-Pruning.jpg b/models/Reference/OPPOer--Qwen-Image-Pruning.jpg new file mode 100644 index 000000000..f78f4b293 Binary files /dev/null and b/models/Reference/OPPOer--Qwen-Image-Pruning.jpg differ diff --git a/models/Reference/Qwen--Qwen-Image-Edit-2509.jpg b/models/Reference/Qwen--Qwen-Image-Edit-2509.jpg new file mode 100644 index 000000000..6209b7ba0 Binary files /dev/null and b/models/Reference/Qwen--Qwen-Image-Edit-2509.jpg differ diff --git a/models/Reference/Qwen-Image-13B.jpg b/models/Reference/Qwen-Image-13B.jpg new file mode 100644 index 000000000..9198264fd Binary files /dev/null and b/models/Reference/Qwen-Image-13B.jpg differ diff --git a/models/Reference/Qwen-Image-Distill.jpg b/models/Reference/Qwen-Image-Distill.jpg deleted file mode 100644 index ac05ebcee..000000000 Binary files a/models/Reference/Qwen-Image-Distill.jpg and /dev/null differ diff --git a/models/Reference/X-Omni--X-Omni-SFT.jpg b/models/Reference/X-Omni--X-Omni-SFT.jpg new file mode 100644 index 000000000..b2394b57b Binary files /dev/null and b/models/Reference/X-Omni--X-Omni-SFT.jpg differ diff --git a/models/Reference/briaai--BRIA-3.2.jpg b/models/Reference/briaai--BRIA-3.2.jpg index dc7067ec5..91476ba26 100644 Binary files a/models/Reference/briaai--BRIA-3.2.jpg and b/models/Reference/briaai--BRIA-3.2.jpg differ diff --git a/models/Reference/fal--AuraFlow-v0.3.jpg b/models/Reference/fal--AuraFlow-v0.3.jpg index 1ce8fe728..beff71e97 100644 Binary files a/models/Reference/fal--AuraFlow-v0.3.jpg and b/models/Reference/fal--AuraFlow-v0.3.jpg differ diff --git a/models/Reference/juggernautXL_v9Rundiffusionphoto2.jpg b/models/Reference/juggernautXL_juggXIByRundiffusion.jpg similarity index 100% rename from models/Reference/juggernautXL_v9Rundiffusionphoto2.jpg rename to models/Reference/juggernautXL_juggXIByRundiffusion.jpg diff --git a/models/Reference/juggernautXL_v9Rdphoto2Lightning.jpg b/models/Reference/juggernautXL_juggXILightningByRD.jpg similarity index 100% rename from models/Reference/juggernautXL_v9Rdphoto2Lightning.jpg rename to models/Reference/juggernautXL_juggXILightningByRD.jpg diff --git a/models/Reference/kandinsky-community--kandinsky-2-1.jpg b/models/Reference/kandinsky-community--kandinsky-2-1.jpg index 2fb28d1a4..bebbd6379 100644 Binary files a/models/Reference/kandinsky-community--kandinsky-2-1.jpg and b/models/Reference/kandinsky-community--kandinsky-2-1.jpg differ diff --git a/models/Reference/kandinsky-community--kandinsky-2-2-decoder.jpg b/models/Reference/kandinsky-community--kandinsky-2-2-decoder.jpg index 743fe024c..75f7ba309 100644 Binary files a/models/Reference/kandinsky-community--kandinsky-2-2-decoder.jpg and b/models/Reference/kandinsky-community--kandinsky-2-2-decoder.jpg differ diff --git a/models/Reference/kandinsky-community--kandinsky-3.jpg b/models/Reference/kandinsky-community--kandinsky-3.jpg index f54af2b8c..933463d44 100644 Binary files a/models/Reference/kandinsky-community--kandinsky-3.jpg and b/models/Reference/kandinsky-community--kandinsky-3.jpg differ diff --git a/models/Reference/linoyts--Wan2.2-VACE-Fun-14B-diffusers.jpg b/models/Reference/linoyts--Wan2.2-VACE-Fun-14B-diffusers.jpg new file mode 100644 index 000000000..dbde7389b Binary files /dev/null and b/models/Reference/linoyts--Wan2.2-VACE-Fun-14B-diffusers.jpg differ diff --git a/models/Reference/playgroundai--playground-v2_5-1024px-aesthetic.jpg b/models/Reference/playgroundai--playground-v2.5-1024px-aesthetic.jpg similarity index 100% rename from models/Reference/playgroundai--playground-v2_5-1024px-aesthetic.jpg rename to models/Reference/playgroundai--playground-v2.5-1024px-aesthetic.jpg diff --git a/models/Reference/segmind--SSD-1B.jpg b/models/Reference/segmind--SSD-1B.jpg index 51907db27..11a96eedb 100644 Binary files a/models/Reference/segmind--SSD-1B.jpg and b/models/Reference/segmind--SSD-1B.jpg differ diff --git a/models/Reference/segmind--SegMoE-4x2-v0.jpg b/models/Reference/segmind--SegMoE-4x2-v0.jpg index 93903538c..3e9ba4ea3 100644 Binary files a/models/Reference/segmind--SegMoE-4x2-v0.jpg and b/models/Reference/segmind--SegMoE-4x2-v0.jpg differ diff --git a/models/Reference/segmind--SegMoE-SD-4x2-v0.jpg b/models/Reference/segmind--SegMoE-SD-4x2-v0.jpg index 7d7989ec2..24b738c02 100644 Binary files a/models/Reference/segmind--SegMoE-SD-4x2-v0.jpg and b/models/Reference/segmind--SegMoE-SD-4x2-v0.jpg differ diff --git a/models/Reference/segmind--Segmind-Vega.jpg b/models/Reference/segmind--Segmind-Vega.jpg index 98a5ec811..5204f1712 100644 Binary files a/models/Reference/segmind--Segmind-Vega.jpg and b/models/Reference/segmind--Segmind-Vega.jpg differ diff --git a/models/Reference/segmind--tiny-sd.jpg b/models/Reference/segmind--tiny-sd.jpg index 375f697aa..953407d56 100644 Binary files a/models/Reference/segmind--tiny-sd.jpg and b/models/Reference/segmind--tiny-sd.jpg differ diff --git a/models/Reference/stabilityai--sdxl-turbo.jpg b/models/Reference/stabilityai--sdxl-turbo.jpg new file mode 100644 index 000000000..122f3678a Binary files /dev/null and b/models/Reference/stabilityai--sdxl-turbo.jpg differ diff --git a/models/Reference/stabilityai--stable-diffusion-2-1-base.jpg b/models/Reference/stabilityai--stable-diffusion-2-1-base.jpg index ab0e655f2..9912ea894 100644 Binary files a/models/Reference/stabilityai--stable-diffusion-2-1-base.jpg and b/models/Reference/stabilityai--stable-diffusion-2-1-base.jpg differ diff --git a/models/Reference/stabilityai--stable-diffusion-2-1.jpg b/models/Reference/stabilityai--stable-diffusion-2-1.jpg index 42205ad67..0560d5bfd 100644 Binary files a/models/Reference/stabilityai--stable-diffusion-2-1.jpg and b/models/Reference/stabilityai--stable-diffusion-2-1.jpg differ diff --git a/models/Reference/sd_xl_base_1.0.jpg b/models/Reference/stabilityai--stable-diffusion-xl-base-1.0.jpg similarity index 100% rename from models/Reference/sd_xl_base_1.0.jpg rename to models/Reference/stabilityai--stable-diffusion-xl-base-1.0.jpg diff --git a/models/Reference/tempest-by-vlad-base.jpg b/models/Reference/tempest-by-vlad-base.jpg deleted file mode 100644 index 0d48f0a62..000000000 Binary files a/models/Reference/tempest-by-vlad-base.jpg and /dev/null differ diff --git a/models/Reference/tempest-by-vlad-hyper.jpg b/models/Reference/tempest-by-vlad-hyper.jpg deleted file mode 100644 index c2ffc23e8..000000000 Binary files a/models/Reference/tempest-by-vlad-hyper.jpg and /dev/null differ diff --git a/models/Reference/tempestByVlad_baseV01.jpg b/models/Reference/tempestByVlad_baseV01.jpg new file mode 100644 index 000000000..2db7f03da Binary files /dev/null and b/models/Reference/tempestByVlad_baseV01.jpg differ diff --git a/models/Reference/tempestByVlad_hyperV01.jpg b/models/Reference/tempestByVlad_hyperV01.jpg new file mode 100644 index 000000000..dc28c29d8 Binary files /dev/null and b/models/Reference/tempestByVlad_hyperV01.jpg differ diff --git a/models/Reference/v1-5-pruned-fp16-emaonly.jpg b/models/Reference/v1-5-pruned-fp16-emaonly.jpg index 78a9c7e11..8dc8df64b 100644 Binary files a/models/Reference/v1-5-pruned-fp16-emaonly.jpg and b/models/Reference/v1-5-pruned-fp16-emaonly.jpg differ diff --git a/models/Reference/vladmandic--flux.1-dev-SRPO.jpg b/models/Reference/vladmandic--flux.1-dev-SRPO.jpg new file mode 100644 index 000000000..986483517 Binary files /dev/null and b/models/Reference/vladmandic--flux.1-dev-SRPO.jpg differ diff --git a/modules/api/docs.py b/modules/api/docs.py index f384a1328..1eac8226e 100644 --- a/modules/api/docs.py +++ b/modules/api/docs.py @@ -41,20 +41,18 @@ def get_swagger_ui_html(*, if oauth2_redirect_url: html += f"oauth2RedirectUrl: window.location.origin + '{oauth2_redirect_url}'," html += """ - presets: [ - SwaggerUIBundle.presets.apis, - SwaggerUIBundle.SwaggerUIStandalonePreset - ], - })""" + presets: [ + SwaggerUIBundle.presets.apis, + SwaggerUIBundle.SwaggerUIStandalonePreset + ], + })""" if init_oauth: - html += f""" - ui.initOAuth({json.dumps(jsonable_encoder(init_oauth))}) - """ + html += f"ui.initOAuth({json.dumps(jsonable_encoder(init_oauth))})" html += """ - - - - """ + + + + """ return HTMLResponse(html) @@ -74,8 +72,9 @@ def create_docs(app: FastAPI): title=f'{app.title}: Swagger UI', openapi_url=app.openapi_url, swagger_favicon_url='/file=html/favicon.svg', + swagger_css_url='/file=html/swagger.css', swagger_ui_parameters=swagger_ui_parameters, - swagger_extra_css_url='file=html/swagger.css', + # swagger_extra_css_url='file=html/swagger.css', ) # res = inject_css(html.content, 'html/swagger.css') return res diff --git a/modules/cachedit.py b/modules/cachedit.py new file mode 100644 index 000000000..057b23804 --- /dev/null +++ b/modules/cachedit.py @@ -0,0 +1,64 @@ +import os +from installer import install +from modules import shared + + +def apply_cache_dit(pipe): + if not shared.opts.cache_dit_enabled: + return + install('git+https://github.com/vipshop/cache-dit', 'cache_dit') + os.environ.setdefault("CACHE_DIT_LOG_LEVEL", "error") + try: + import cache_dit + except Exception as e: + shared.log.error(f'Cache-DIT: {e}') + return + _, supported = cache_dit.supported_pipelines() + supported = [s.replace('*', '') for s in supported] + if not any(pipe.__class__.__name__.startswith(s) for s in supported): + shared.log.error(f'Cache-DiT: pipeline={pipe.__class__.__name__} unsupported') + return + + if getattr(pipe, 'has_cache_dit', False): + unapply_cache_dir(pipe) + + config_args = {} + if shared.opts.cache_dit_fcompute >= 0: + config_args['Fn_compute_blocks'] = int(shared.opts.cache_dit_fcompute) + if shared.opts.cache_dit_bcompute >= 0: + config_args['Bn_compute_blocks'] = int(shared.opts.cache_dit_bcompute) + if shared.opts.cache_dit_threshold >= 0: + config_args['residual_diff_threshold'] = float(shared.opts.cache_dit_threshold) + if shared.opts.cache_dit_warmup >= 0: + config_args['max_warmup_steps'] = int(shared.opts.cache_dit_warmup) + cache_config = cache_dit.BasicCacheConfig(**config_args) + if shared.opts.cache_dit_calibrator == "TaylorSeer": + calibrator_config = cache_dit.TaylorSeerCalibratorConfig(taylorseer_order=1) + elif shared.opts.cache_dit_calibrator == "FoCa": + calibrator_config = cache_dit.FoCaCalibratorConfig() + else: + calibrator_config = None + shared.log.info(f'Apply Cache-DiT: config="{cache_config.strify()}" calibrator="{calibrator_config.strify() if calibrator_config else "None"}"') + try: + cache_dit.enable_cache( + pipe, + cache_config=cache_config, + calibrator_config=calibrator_config, + ) + shared.sd_model.has_cache_dit = True + except Exception as e: + shared.log.error(f'Cache-DiT: {e}') + return + + +def unapply_cache_dir(pipe): + if not shared.opts.cache_dit_enabled or not getattr(pipe, 'has_cache_dit', False): + return + try: + import cache_dit + # stats = cache_dit.summary(pipe) + # shared.log.critical(f'Unapply Cache-DiT: {stats}') + cache_dit.disable_cache(pipe) + pipe.has_cache_dit = False + except Exception: + return diff --git a/modules/civitai/api_civitai.py b/modules/civitai/api_civitai.py index 92add902f..4041f3f36 100644 --- a/modules/civitai/api_civitai.py +++ b/modules/civitai/api_civitai.py @@ -54,6 +54,15 @@ def get_civitai( return JSONResponse(content=[], status_code=200) +def post_civitai(page:str=None): + from modules.civitai import metadata_civitai + result = [] + for r in metadata_civitai.civit_search_metadata(title=page, raw=True): + result = r # get the last yielded result + return result + + def register_api(): from modules.shared import api api.add_api_route("/sdapi/v1/civitai", get_civitai, methods=["GET"], response_model=list) + api.add_api_route("/sdapi/v1/civitai", post_civitai, methods=["POST"], response_model=list) diff --git a/modules/civitai/download_civitai.py b/modules/civitai/download_civitai.py index 508bd0a8f..d12a00830 100644 --- a/modules/civitai/download_civitai.py +++ b/modules/civitai/download_civitai.py @@ -103,10 +103,11 @@ def download_civit_model_thread(model_name: str, model_url: str, model_path: str if os.path.isfile(temp_file): starting_pos = os.path.getsize(temp_file) headers['Range'] = f'bytes={starting_pos}-' - if token is None or len(token) == 0: - token = shared.opts.civitai_token - if token is not None and len(token) > 0: - headers['Authorization'] = f'Bearer {token}' + if 'civit' in model_url.lower(): # downloader can be used for other urls too + if token is None or len(token) == 0: + token = shared.opts.civitai_token + if (token is not None) and (len(token) > 0): + headers['Authorization'] = f'Bearer {token}' r = shared.req(model_url, headers=headers, stream=True) total_size = int(r.headers.get('content-length', 0)) diff --git a/modules/civitai/metadata_civitai.py b/modules/civitai/metadata_civitai.py index 6d4315860..fab22e78f 100644 --- a/modules/civitai/metadata_civitai.py +++ b/modules/civitai/metadata_civitai.py @@ -26,7 +26,7 @@ class CivitModel: self.status = 'Not found' -def civit_update_metadata(): +def civit_update_metadata(raw:bool=False): def create_update_metadata_table(rows: list[CivitModel]): html = """ @@ -102,8 +102,8 @@ def civit_update_metadata(): model.status = 'Update available' break results.append(model) - yield create_update_metadata_table(results) - return create_update_metadata_table(results) + yield results if raw else create_update_metadata_table(results) + yield results if raw else create_update_metadata_table(results) def civit_search_model(name, tag, model_type): @@ -224,7 +224,7 @@ def atomic_civit_search_metadata(item, results): results.append(result) -def civit_search_metadata(title: str = None): +def civit_search_metadata(title: str = None, raw: bool = False): def create_search_metadata_table(rows): html = """
@@ -258,7 +258,7 @@ def civit_search_metadata(title: str = None): re_skip = [r.strip() for r in opts.extra_networks_scan_skip.split(',') if len(r.strip()) > 0] for page in get_pages(): if type(title) == str: - if page.title != title: + if page.title.lower() != title.lower(): continue if page.name == 'style' or page.name == 'wildcards': continue @@ -278,8 +278,8 @@ def civit_search_metadata(title: str = None): future_items[executor.submit(atomic_civit_search_metadata, fn, results)] = fn for future in concurrent.futures.as_completed(future_items): future.result() - yield create_search_metadata_table(results) + yield results if raw else create_search_metadata_table(results) t1 = time.time() log.debug(f'CivitAI search metadata: scanned={scanned} skipped={skipped} time={t1-t0:.2f}') - yield create_search_metadata_table(results) + yield results if raw else create_search_metadata_table(results) diff --git a/modules/control/run.py b/modules/control/run.py index 6d1b885b7..b62cad89f 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -255,6 +255,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg prompt: str = '', negative_prompt: str = '', styles: List[str] = [], steps: int = 20, sampler_index: int = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, + guidance_name: str = 'Default', guidance_scale: float = 6.0, guidance_rescale: float = 0.0, guidance_start: float = 0.0, guidance_stop: float = 1.0, cfg_scale: float = 6.0, clip_skip: float = 1.0, image_cfg_scale: float = 6.0, diffusers_guidance_rescale: float = 0.7, pag_scale: float = 0.0, pag_adaptive: float = 0.5, cfg_end: float = 1.0, vae_type: str = 'Full', tiling: bool = False, hidiffusion: bool = False, detailer_enabled: bool = True, detailer_prompt: str = '', detailer_negative: str = '', detailer_steps: int = 10, detailer_strength: float = 0.3, detailer_resolution: int = 1024, @@ -306,7 +307,13 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg seed_resize_from_h = seed_resize_from_h, seed_resize_from_w = seed_resize_from_w, denoising_strength = denoising_strength, - # advanced + # modular guidance + guidance_name = guidance_name, + guidance_scale = guidance_scale, + guidance_rescale = guidance_rescale, + guidance_start = guidance_start, + guidance_stop = guidance_stop, + # legacy guidance cfg_scale = cfg_scale, cfg_end = cfg_end, clip_skip = clip_skip, @@ -314,6 +321,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg diffusers_guidance_rescale = diffusers_guidance_rescale, pag_scale = pag_scale, pag_adaptive = pag_adaptive, + # advanced vae_type = vae_type, tiling = tiling, hidiffusion = hidiffusion, @@ -380,6 +388,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg ) p.state = state p.is_tile = False + p.init_control = inits or [] p.orig_init_images = inputs # TODO modernui: monkey-patch for missing tabs.select event @@ -396,9 +405,9 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg # hires/refine defined outside of main init vae_scale_factor = sd_vae.get_vae_scale_factor() if p.enable_hr and (p.hr_resize_x == 0 or p.hr_resize_y == 0): - p.hr_upscale_to_x, p.hr_upscale_to_y = vae_scale_factor * int(p.width_before * p.hr_scale / vae_scale_factor), vae_scale_factor * int(p.height_before * p.hr_scale / vae_scale_factor) + p.hr_upscale_to_x, p.hr_upscale_to_y = int(vae_scale_factor * int(p.width_before * p.hr_scale / vae_scale_factor)), int(vae_scale_factor * int(p.height_before * p.hr_scale / vae_scale_factor)) elif p.enable_hr and (p.hr_upscale_to_x == 0 or p.hr_upscale_to_y == 0): - p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.hr_resize_x / vae_scale_factor), vae_scale_factor * int(p.hr_resize_y / vae_scale_factor) + p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.hr_resize_x / vae_scale_factor), int(vae_scale_factor * int(p.hr_resize_y / vae_scale_factor)) global p_extra_args # pylint: disable=global-statement for k, v in p_extra_args.items(): diff --git a/modules/devices.py b/modules/devices.py index abe979c17..a98b53d02 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -49,6 +49,10 @@ def has_xpu() -> bool: return bool(hasattr(torch, 'xpu') and torch.xpu.is_available()) +def has_rocm() -> bool: + return bool(torch.version.hip is not None and torch.cuda.is_available()) + + def has_zluda() -> bool: if not cuda_ok: return False diff --git a/modules/extras.py b/modules/extras.py index 2914d6c1e..4f282653c 100644 --- a/modules/extras.py +++ b/modules/extras.py @@ -39,24 +39,24 @@ def run_modelmerger(id_task, **kwargs): # pylint: disable=unused-argument return [*[gr.update() for _ in range(4)], message] kwargs["models"] = { - "model_a": sd_models.get_closet_checkpoint_match(kwargs.get("primary_model_name", None)).filename, - "model_b": sd_models.get_closet_checkpoint_match(kwargs.get("secondary_model_name", None)).filename, + "model_a": sd_models.get_closest_checkpoint_match(kwargs.get("primary_model_name", None)).filename, + "model_b": sd_models.get_closest_checkpoint_match(kwargs.get("secondary_model_name", None)).filename, } if kwargs.get("primary_model_name", None) in [None, 'None']: return fail("Failed: Merging requires a primary model.") - primary_model_info = sd_models.get_closet_checkpoint_match(kwargs.get("primary_model_name", None)) + primary_model_info = sd_models.get_closest_checkpoint_match(kwargs.get("primary_model_name", None)) if kwargs.get("secondary_model_name", None) in [None, 'None']: return fail("Failed: Merging requires a secondary model.") - secondary_model_info = sd_models.get_closet_checkpoint_match(kwargs.get("secondary_model_name", None)) + secondary_model_info = sd_models.get_closest_checkpoint_match(kwargs.get("secondary_model_name", None)) if kwargs.get("tertiary_model_name", None) in [None, 'None'] and kwargs.get("merge_mode", None) in merge_utils.TRIPLE_METHODS: return fail(f"Failed: Interpolation method ({kwargs.get('merge_mode', None)}) requires a tertiary model.") - tertiary_model_info = sd_models.get_closet_checkpoint_match(kwargs.get("tertiary_model_name", None)) if kwargs.get("merge_mode", None) in merge_utils.TRIPLE_METHODS else None + tertiary_model_info = sd_models.get_closest_checkpoint_match(kwargs.get("tertiary_model_name", None)) if kwargs.get("merge_mode", None) in merge_utils.TRIPLE_METHODS else None del kwargs["primary_model_name"] del kwargs["secondary_model_name"] if kwargs.get("tertiary_model_name", None) is not None: - kwargs["models"] |= {"model_c": sd_models.get_closet_checkpoint_match(kwargs.get("tertiary_model_name", None)).filename} + kwargs["models"] |= {"model_c": sd_models.get_closest_checkpoint_match(kwargs.get("tertiary_model_name", None)).filename} del kwargs["tertiary_model_name"] if kwargs.get("alpha_base", None) and kwargs.get("alpha_in_blocks", None) and kwargs.get("alpha_mid_block", None) and kwargs.get("alpha_out_blocks", None): @@ -204,7 +204,7 @@ def run_model_modules(model_type:str, model_name:str, custom_name:str, if len(custom_name) == 0: yield msg("output name is required", err=True) return - checkpoint_info = sd_models.get_closet_checkpoint_match(model_name) + checkpoint_info = sd_models.get_closest_checkpoint_match(model_name) if checkpoint_info is None: yield msg("input model not found", err=True) return diff --git a/modules/face/insightface.py b/modules/face/insightface.py index b69648904..99be2d32e 100644 --- a/modules/face/insightface.py +++ b/modules/face/insightface.py @@ -10,11 +10,9 @@ instightface_mp = None def get_app(mp_name, threshold=0.5, resolution=640): global insightface_app, instightface_mp # pylint: disable=global-statement - from installer import install, installed, install_pydantic + from installer import install, installed, install_insightface if not installed('insightface', reload=False, quiet=True): - install('git+https://github.com/deepinsight/insightface@554a05561cb71cfebb4e012dfea48807f845a0c2#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches - install('albumentations==1.4.3', ignore=False, reinstall=True) - install_pydantic() + install_insightface() if not installed('ip_adapter', reload=False, quiet=True): install('git+https://github.com/tencent-ailab/IP-Adapter.git', 'ip_adapter', ignore=False) diff --git a/modules/framepack/framepack_load.py b/modules/framepack/framepack_load.py index 0e80e8485..ed9fdde43 100644 --- a/modules/framepack/framepack_load.py +++ b/modules/framepack/framepack_load.py @@ -124,6 +124,10 @@ def load_model(variant:str=None, pipeline:str=None, text_encoder:str=None, text_ sd_models.unload_model_weights() t0 = time.time() + sd_models.hf_auth_check(model["transformer"]["repo"]) + sd_models.hf_auth_check(model["text_encoder"]["repo"]) + sd_models.hf_auth_check(model["text_encoder_2"]["repo"]) + shared.log.debug(f'FramePack load: module=llm {model["text_encoder"]}') load_args, quant_args = model_quant.get_dit_args({}, module='TE', device_map=True) text_encoder = LlamaModel.from_pretrained(model["text_encoder"]["repo"], subfolder=model["text_encoder"]["subfolder"], cache_dir=shared.opts.hfcache_dir, **load_args, **quant_args) diff --git a/modules/framepack/framepack_vae.py b/modules/framepack/framepack_vae.py index da5144cf7..908378a8b 100644 --- a/modules/framepack/framepack_vae.py +++ b/modules/framepack/framepack_vae.py @@ -85,7 +85,10 @@ def vae_decode(latents, vae_type): elif vae_type == 'Remote': return vae_decode_remote(latents) else: # vae_type == 'Full' - return vae_decode_full(latents) + jobid = shared.state.begin('VAE Decode') + result = vae_decode_full(latents) + shared.state.end(jobid) + return result def vae_encode(image): diff --git a/modules/framepack/framepack_worker.py b/modules/framepack/framepack_worker.py index 1b87334ec..cda5677f8 100644 --- a/modules/framepack/framepack_worker.py +++ b/modules/framepack/framepack_worker.py @@ -54,7 +54,7 @@ def worker( from modules.framepack.pipeline import hunyuan from modules.framepack.pipeline import utils - from modules.framepack.pipeline.k_diffusion_hunyuan import sample_hunyuan + from modules.framepack.pipeline import k_diffusion_hunyuan is_f1 = variant == 'forward-only' total_generated_frames = 0 @@ -65,14 +65,13 @@ def worker( metadata['title'] = 'sdnext framepack' metadata['description'] = f'variant:{variant} seed:{seed} steps:{steps} scale:{cfg_scale} distilled:{cfg_distilled} rescale:{cfg_rescale} shift:{shift} start:{start_weight} end:{end_weight} vision:{vision_weight}' - shared.state.begin('Video') + videojob = shared.state.begin('Video') shared.state.job_count = 1 text_encoder = shared.sd_model.text_encoder text_encoder_2 = shared.sd_model.text_encoder_2 tokenizer = shared.sd_model.tokenizer tokenizer_2 = shared.sd_model.tokenizer_2 - vae = shared.sd_model.vae feature_extractor = shared.sd_model.feature_extractor image_encoder = shared.sd_model.image_processor transformer = shared.sd_model.transformer @@ -196,7 +195,6 @@ def worker( image_encoder_last_hidden_state = vision_encode(input_image, end_image) # Sample loop - shared.state.textinfo = 'Sample' stream.output_queue.push(('progress', (None, 'Start sampling...'))) generator = torch.Generator("cpu").manual_seed(seed) if is_f1: @@ -213,6 +211,7 @@ def worker( llama_vec, llama_vec_n, llama_attention_mask, llama_attention_mask_n, clip_l_pooler, clip_l_pooler_n = text_encode(current_prompt, i=lattent_padding_loop+1) last_prompt = current_prompt + sammplejob = shared.state.begin('Sample') lattent_padding_loop += 1 # shared.log.trace(f'FramePack: op=sample section={lattent_padding_loop}/{len(latent_paddings)} frames={total_generated_frames}/{num_frames*len(latent_paddings)} window={latent_window_size} size={num_frames}') if is_f1: @@ -244,7 +243,7 @@ def worker( transformer.initialize_teacache(enable_teacache=use_teacache, num_steps=steps, rel_l1_thresh=shared.opts.teacache_thresh) t_sample = time.time() - generated_latents = sample_hunyuan( + generated_latents = k_diffusion_hunyuan.sample_hunyuan( transformer=transformer, sampler='unipc', width=width, @@ -274,7 +273,6 @@ def worker( dtype=devices.dtype, callback=step_callback, ) - timer.process.add('sample', time.time()-t_sample) if is_last_section: generated_latents = torch.cat([start_latent.to(generated_latents), generated_latents], dim=2) @@ -287,15 +285,18 @@ def worker( history_latents = torch.cat([generated_latents.to(history_latents), history_latents], dim=2) real_history_latents = history_latents[:, :, :total_generated_latent_frames, :, :] - t_vae = time.time() sd_models.apply_balanced_offload(shared.sd_model) + timer.process.add('sample', time.time()-t_sample) + shared.state.end(sammplejob) + + t_vae = time.time() if history_pixels is None: history_pixels = framepack_vae.vae_decode(real_history_latents, vae_type=vae_type).cpu() else: overlapped_frames = latent_window_size * 4 - 3 if is_f1: section_latent_frames = latent_window_size * 2 - current_pixels = framepack_vae.vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu() + current_pixels = framepack_vae.vae_decode(real_history_latents[:, :, -section_latent_frames:], vae_type=vae_type).cpu() history_pixels = utils.soft_append_bcthw(history_pixels, current_pixels, overlapped_frames) else: section_latent_frames = (latent_window_size * 2 + 1) if is_last_section else (latent_window_size * 2) @@ -321,4 +322,4 @@ def worker( stream.output_queue.push(('end', None)) t1 = time.time() shared.log.info(f'Processed: frames={total_generated_frames} fps={total_generated_frames/(t1-t0):.2f} its={(shared.state.sampling_step)/(t1-t0):.2f} time={t1-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') - shared.state.end() + shared.state.end(videojob) diff --git a/modules/framepack/pipeline/uni_pc_fm.py b/modules/framepack/pipeline/uni_pc_fm.py index 6ce7c15ce..2066cd8e3 100644 --- a/modules/framepack/pipeline/uni_pc_fm.py +++ b/modules/framepack/pipeline/uni_pc_fm.py @@ -5,7 +5,7 @@ import torch - +import numpy as np from tqdm.auto import trange @@ -13,6 +13,36 @@ def expand_dims(v, dims): return v[(...,) + (None,) * (dims - 1)] +torch_linalg_solve = None + + +def test_solver(): + from modules import devices, shared + try: + a = torch.randn(50, 50).to(device=devices.device, dtype=torch.float32) + b = torch.randn(50, 2).to(device=devices.device, dtype=torch.float32) + _x = torch.linalg.solve(a, b) + return True + except Exception as e: + shared.log.debug(f'FramePack: solver=cpu {e}') + return False + + +def linalg_solve(A, B, device): + global torch_linalg_solve # pylint: disable=global-statement + if torch_linalg_solve is None: + torch_linalg_solve = test_solver() + if torch_linalg_solve: + X = torch.linalg.solve(A, B) + return X + else: + A_np = A.float().cpu().numpy() + B_np = B.float().cpu().numpy() + X_np = np.linalg.solve(A_np, B_np) + X = torch.from_numpy(X_np).to(device=device, dtype=A.dtype) + return X + + class FlowMatchUniPC: def __init__(self, model, extra_args, variant='bh1'): self.model = model @@ -78,7 +108,7 @@ class FlowMatchUniPC: if order == 2: rhos_p = torch.tensor([0.5], device=b.device) else: - rhos_p = torch.linalg.solve(R[:-1, :-1], b[:-1]) + rhos_p = linalg_solve(R[:-1, :-1], b[:-1], x.device) else: D1s = None rhos_p = None @@ -86,7 +116,7 @@ class FlowMatchUniPC: if order == 1: rhos_c = torch.tensor([0.5], device=b.device) else: - rhos_c = torch.linalg.solve(R, b) + rhos_c = linalg_solve(R, b, x.device) x_t_ = expand_dims(t / t_prev_0, dims) * x - expand_dims(h_phi_1, dims) * model_prev_0 diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py index a2f38b94e..1427e53cc 100644 --- a/modules/generation_parameters_copypaste.py +++ b/modules/generation_parameters_copypaste.py @@ -229,6 +229,11 @@ def connect_paste(button, local_paste_fields, input_comp, override_settings_comp res.append(v) applied[key] = v else: + if isinstance(v, str) and v.strip() == '' and key in {'Prompt', 'Negative prompt'}: + debug(f'Paste skip empty: "{key}"') + res.append(gr.update()) + skipped[key] = v + continue if should_skip(key): debug(f'Paste skip: "{key}"="{v}"') res.append(gr.update()) diff --git a/modules/images_namegen.py b/modules/images_namegen.py index c8971eff2..849450d5b 100644 --- a/modules/images_namegen.py +++ b/modules/images_namegen.py @@ -1,5 +1,6 @@ import re import os +import time import uuid import string import hashlib @@ -15,6 +16,7 @@ re_pattern_arg = re.compile(r"(.*)<([^>]*)>$") re_attention = re.compile(r'[\(*\[*](\w+)(:\d+(\.\d+))?[\)*\]*]|') re_network = re.compile(r'\<\w+:(\w+)(:\d+(\.\d+))?\>|') re_brackets = re.compile(r'[\([{})\]]') +re_leading_seq = re.compile(r'^(0*\d+)(?=[-_.\s]|$)') seq = 0 NOTHING = object() @@ -33,6 +35,7 @@ class FilenameGenerator: 'hash': lambda self: self.image_hash(), 'image_hash': lambda self: self.image_hash(), 'timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp), + 'epoch': lambda self: int(time.time()), 'job_timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp), 'model': lambda self: shared.sd_model.sd_checkpoint_info.title if shared.sd_loaded and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None else '', @@ -183,12 +186,24 @@ class FilenameGenerator: debug(f'Filename sanitize: input="{filename}" parts={parts} output="{fn}" ext={ext} max={max_length} len={len(fn)}') return fn + def safe_int(self, s): + try: + return int(s) + except (ValueError, TypeError): + return 0 + def sequence(self, fn): global seq # pylint: disable=global-statement x = fn dirname = os.path.dirname(fn) if seq == 0: - seq = len(os.listdir(dirname)) if os.path.exists(dirname) and os.path.isdir(dirname) else 0 + files = os.listdir(dirname) if os.path.exists(dirname) and os.path.isdir(dirname) else [] + files = [f for f in files if os.path.isfile(os.path.join(dirname, f))] + seq_files = len(files) + seq_nums = [re_leading_seq.match(f) for f in files] + seq_nums = [self.safe_int(m.group(1)) for m in seq_nums if m is not None] + seq_num = max(seq_nums) if len(seq_nums) > 0 else 0 + seq = max(seq_files, seq_num) if shared.opts.save_images_add_number or '[seq]' in fn: if '[seq]' not in fn: fn = os.path.join(os.path.dirname(fn), f"[seq]-{os.path.basename(fn)}") diff --git a/modules/img2img.py b/modules/img2img.py index 19da4b92f..4809d9fa0 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -161,10 +161,8 @@ def img2img(id_task: str, state: str, mode: int, vae_type, tiling, hidiffusion, detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, n_iter, batch_size, - cfg_scale, image_cfg_scale, - diffusers_guidance_rescale, - pag_scale, pag_adaptive, - cfg_end, + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, + cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, refiner_start, clip_skip, denoising_strength, @@ -255,6 +253,11 @@ def img2img(id_task: str, state: str, mode: int, batch_size=batch_size, n_iter=n_iter, steps=steps, + guidance_name=guidance_name, + guidance_scale=guidance_scale, + guidance_rescale=guidance_rescale, + guidance_start=guidance_start, + guidance_stop=guidance_stop, cfg_scale=cfg_scale, cfg_end=cfg_end, clip_skip=clip_skip, diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index 93f94d280..0a4e19389 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -152,7 +152,7 @@ def ipex_init(): # pylint: disable=too-many-statements # torch._int_mm directly uses onednn quantized matmul # onednn qlinear is a wrapper around onednn quantized matmul if hasattr(torch.ops, "onednn") and hasattr(torch.ops.onednn, "qlinear_pointwise"): - def onednn_mm(x: torch.Tensor, y: torch.Tensor): + def onednn_mm(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: # supports int8, fp32, fp16, and bf16 matmul with accumulation using a different dtype # int8 matmul with onednn is slower than 16 bit with dim_size < 4096 return torch.ops.onednn.qlinear_pointwise.default(x, 1.0, 0, y, torch.ones(1, device=y.device), torch.zeros(1, device=y.device), None, 1.0, 0, torch.float32, "none", [], "none") @@ -165,7 +165,7 @@ def ipex_init(): # pylint: disable=too-many-statements pass # Memory: - if 'linux' in sys.platform and "WSL2" in os.popen("uname -a").read(): + if "linux" in sys.platform and "WSL2" in os.popen("uname -a").read(): torch.xpu.empty_cache = lambda: None torch.cuda.empty_cache = torch.xpu.empty_cache diff --git a/modules/intel/ipex/attention.py b/modules/intel/ipex/attention.py index c3e8ef8ed..aacccca75 100644 --- a/modules/intel/ipex/attention.py +++ b/modules/intel/ipex/attention.py @@ -8,8 +8,8 @@ from functools import cache, wraps # ARC GPUs can't allocate more than 4GB to a single block so we slice the attention layers -dynamic_attention_slice_rate = float(os.environ.get('IPEX_SDPA_SLICE_TRIGGER_RATE', 1)) -dynamic_attention_trigger_rate = float(os.environ.get('IPEX_ATTENTION_SLICE_RATE', 0.5)) +dynamic_attention_slice_rate = float(os.environ.get("IPEX_SDPA_SLICE_TRIGGER_RATE", "1")) +dynamic_attention_trigger_rate = float(os.environ.get("IPEX_ATTENTION_SLICE_RATE", "0.5")) # Find something divisible with the input_tokens @cache diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index 9f2c5a542..8e8961476 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -1,11 +1,9 @@ -from typing import Optional - import os from functools import wraps from contextlib import nullcontext import torch import numpy as np -from modules import devices, errors +from modules import devices torch_version = torch.__version__[:4] @@ -16,38 +14,25 @@ torch_version[0], torch_version[1] = int(torch_version[0]), int(torch_version[1] device_supports_fp64 = torch.xpu.has_fp64_dtype() if hasattr(torch.xpu, "has_fp64_dtype") else torch.xpu.get_device_properties(devices.device).has_fp64 -if os.environ.get('IPEX_FORCE_ATTENTION_SLICE', '0') == '0': - if torch_version[0] > 2 or (torch_version[0] == 2 and torch_version[1] >= 7): - use_dynamic_attention = False # torch 2.7 has flash atten support - else: - use_dynamic_attention = True -else: - use_dynamic_attention = bool(os.environ.get('IPEX_FORCE_ATTENTION_SLICE', '0') == '1') - # pylint: disable=protected-access, missing-function-docstring, line-too-long, unnecessary-lambda, no-else-return -class DummyDataParallel(torch.nn.Module): # pylint: disable=missing-class-docstring, unused-argument, too-few-public-methods - def __new__(cls, module, device_ids=None, output_device=None, dim=0): # pylint: disable=unused-argument - if isinstance(device_ids, list) and len(device_ids) > 1: - errors.log.error("IPEX backend doesn't support DataParallel on multiple XPU devices") - return module.to(devices.device) - -def return_null_context(*args, **kwargs): # pylint: disable=unused-argument - return nullcontext() @property def is_cuda(self): return self.device.type == "xpu" or self.device.type == "cuda" + def check_device_type(device, device_type: str) -> bool: if device is None or type(device) not in {str, int, torch.device}: return False else: return bool(torch.device(device).type == device_type) + def check_cuda(device) -> bool: return bool(isinstance(device, int) or check_device_type(device, "cuda")) + def return_xpu(device): # keep the device instance type, aka return string if the input is string return devices.device if device is None else f"xpu:{device.split(':')[-1]}" if isinstance(device, str) and ":" in device else f"xpu:{device}" if isinstance(device, int) else torch.device(f"xpu:{device.index}" if device.index is not None else "xpu") if isinstance(device, torch.device) else "xpu" @@ -95,8 +80,8 @@ def torch_get_autocast_dtype(device_type=None): # IPEX 2.5 and above has partial support but doesn't really work most of the time. original_interpolate = torch.nn.functional.interpolate @wraps(torch.nn.functional.interpolate) -def interpolate(tensor, size=None, scale_factor=None, mode='nearest', align_corners=None, recompute_scale_factor=None, antialias=False): # pylint: disable=too-many-arguments - if mode in {'bicubic', 'bilinear'}: +def interpolate(tensor, size=None, scale_factor=None, mode="nearest", align_corners=None, recompute_scale_factor=None, antialias=False): # pylint: disable=too-many-arguments + if mode in {"bicubic", "bilinear"}: return_device = tensor.device return_dtype = tensor.dtype return original_interpolate(tensor.to("cpu", dtype=torch.float32), size=size, scale_factor=scale_factor, mode=mode, @@ -106,6 +91,32 @@ def interpolate(tensor, size=None, scale_factor=None, mode='nearest', align_corn align_corners=align_corners, recompute_scale_factor=recompute_scale_factor, antialias=antialias) +# SwinIR BF16: +original_functional_pad = torch.nn.functional.pad +@wraps(torch.nn.functional.pad) +def functional_pad(input, pad, mode="constant", value=None): + if mode == "reflect" and input.dtype == torch.bfloat16: + return original_functional_pad(input.to(torch.float32), pad, mode=mode, value=value).to(dtype=torch.bfloat16) + else: + return original_functional_pad(input, pad, mode=mode, value=value) + + +# Diffusers FreeU +original_fft_fftn = torch.fft.fftn +@wraps(torch.fft.fftn) +def fft_fftn(input, s=None, dim=None, norm=None, *, out=None): + return_dtype = input.dtype + return original_fft_fftn(input.to(dtype=torch.float32), s=s, dim=dim, norm=norm, out=out).to(dtype=return_dtype) + + +# Diffusers FreeU +original_fft_ifftn = torch.fft.ifftn +@wraps(torch.fft.ifftn) +def fft_ifftn(input, s=None, dim=None, norm=None, *, out=None): + return_dtype = input.dtype + return original_fft_ifftn(input.to(dtype=torch.float32), s=s, dim=dim, norm=norm, out=out).to(dtype=return_dtype) + + # Diffusers Float64 (Alchemist GPUs doesn't support 64 bit): original_from_numpy = torch.from_numpy @wraps(torch.from_numpy) @@ -115,6 +126,7 @@ def from_numpy(ndarray): else: return original_from_numpy(ndarray) + original_as_tensor = torch.as_tensor @wraps(torch.as_tensor) def as_tensor(data, dtype=None, device=None): @@ -126,120 +138,6 @@ def as_tensor(data, dtype=None, device=None): return original_as_tensor(data, dtype=dtype, device=device) -if not use_dynamic_attention: - original_scaled_dot_product_attention = torch.nn.functional.scaled_dot_product_attention -else: - # 32 bit attention workarounds for Alchemist: - try: - from .attention import dynamic_scaled_dot_product_attention as original_scaled_dot_product_attention - except ImportError: - original_scaled_dot_product_attention = torch.nn.functional.scaled_dot_product_attention - -@wraps(torch.nn.functional.scaled_dot_product_attention) -def scaled_dot_product_attention(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor: - if query.dtype != key.dtype: - key = key.to(dtype=query.dtype) - if query.dtype != value.dtype: - value = value.to(dtype=query.dtype) - if attn_mask is not None and query.dtype != attn_mask.dtype: - attn_mask = attn_mask.to(dtype=query.dtype) - if enable_gqa: - kwargs["enable_gqa"] = enable_gqa - result = original_scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs) - if result.dtype != query.dtype: - result = result.to(dtype=query.dtype) - return result - -# Data Type Errors: -original_torch_bmm = torch.bmm -@wraps(torch.bmm) -def torch_bmm(input, mat2, *, out=None): - if input.dtype != mat2.dtype: - mat2 = mat2.to(dtype=input.dtype) - return original_torch_bmm(input, mat2, out=out) - -# Diffusers FreeU -original_fft_fftn = torch.fft.fftn -@wraps(torch.fft.fftn) -def fft_fftn(input, s=None, dim=None, norm=None, *, out=None): - return_dtype = input.dtype - return original_fft_fftn(input.to(dtype=torch.float32), s=s, dim=dim, norm=norm, out=out).to(dtype=return_dtype) - -# Diffusers FreeU -original_fft_ifftn = torch.fft.ifftn -@wraps(torch.fft.ifftn) -def fft_ifftn(input, s=None, dim=None, norm=None, *, out=None): - return_dtype = input.dtype - return original_fft_ifftn(input.to(dtype=torch.float32), s=s, dim=dim, norm=norm, out=out).to(dtype=return_dtype) - -# A1111 FP16 -original_functional_group_norm = torch.nn.functional.group_norm -@wraps(torch.nn.functional.group_norm) -def functional_group_norm(input, num_groups, weight=None, bias=None, eps=1e-05): - if weight is not None and input.dtype != weight.data.dtype: - input = input.to(dtype=weight.data.dtype) - if bias is not None and weight is not None and bias.data.dtype != weight.data.dtype: - bias.data = bias.data.to(dtype=weight.data.dtype) - return original_functional_group_norm(input, num_groups, weight=weight, bias=bias, eps=eps) - -# A1111 BF16 -original_functional_layer_norm = torch.nn.functional.layer_norm -@wraps(torch.nn.functional.layer_norm) -def functional_layer_norm(input, normalized_shape, weight=None, bias=None, eps=1e-05): - if weight is not None and input.dtype != weight.data.dtype: - input = input.to(dtype=weight.data.dtype) - if bias is not None and weight is not None and bias.data.dtype != weight.data.dtype: - bias.data = bias.data.to(dtype=weight.data.dtype) - return original_functional_layer_norm(input, normalized_shape, weight=weight, bias=bias, eps=eps) - -# Training -original_functional_linear = torch.nn.functional.linear -@wraps(torch.nn.functional.linear) -def functional_linear(input, weight, bias=None): - if input.dtype != weight.data.dtype: - input = input.to(dtype=weight.data.dtype) - if bias is not None and bias.data.dtype != weight.data.dtype: - bias.data = bias.data.to(dtype=weight.data.dtype) - return original_functional_linear(input, weight, bias=bias) - -original_functional_conv1d = torch.nn.functional.conv1d -@wraps(torch.nn.functional.conv1d) -def functional_conv1d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): - if input.dtype != weight.data.dtype: - input = input.to(dtype=weight.data.dtype) - if bias is not None and bias.data.dtype != weight.data.dtype: - bias.data = bias.data.to(dtype=weight.data.dtype) - return original_functional_conv1d(input, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups) - -original_functional_conv2d = torch.nn.functional.conv2d -@wraps(torch.nn.functional.conv2d) -def functional_conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): - if input.dtype != weight.data.dtype: - input = input.to(dtype=weight.data.dtype) - if bias is not None and bias.data.dtype != weight.data.dtype: - bias.data = bias.data.to(dtype=weight.data.dtype) - return original_functional_conv2d(input, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups) - -# LTX Video -original_functional_conv3d = torch.nn.functional.conv3d -@wraps(torch.nn.functional.conv3d) -def functional_conv3d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): - if input.dtype != weight.data.dtype: - input = input.to(dtype=weight.data.dtype) - if bias is not None and bias.data.dtype != weight.data.dtype: - bias.data = bias.data.to(dtype=weight.data.dtype) - return original_functional_conv3d(input, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups) - -# SwinIR BF16: -original_functional_pad = torch.nn.functional.pad -@wraps(torch.nn.functional.pad) -def functional_pad(input, pad, mode='constant', value=None): - if mode == 'reflect' and input.dtype == torch.bfloat16: - return original_functional_pad(input.to(torch.float32), pad, mode=mode, value=value).to(dtype=torch.bfloat16) - else: - return original_functional_pad(input, pad, mode=mode, value=value) - - original_torch_tensor = torch.tensor @wraps(torch.tensor) def torch_tensor(data, *args, dtype=None, device=None, **kwargs): @@ -254,20 +152,20 @@ def torch_tensor(data, *args, dtype=None, device=None, **kwargs): dtype = torch.float32 return original_torch_tensor(data, *args, dtype=dtype, device=device, **kwargs) + torch.Tensor.original_Tensor_to = torch.Tensor.to @wraps(torch.Tensor.to) def Tensor_to(self, device=None, *args, **kwargs): + global device_supports_fp64 if check_cuda(device): - if not device_supports_fp64 and kwargs.get("dtype", None) == torch.float64: + device = return_xpu(device) + if not device_supports_fp64: + if kwargs.get("dtype", None) == torch.float64 and ((device is None and self.device.type == "xpu") or (device is not None and torch.device(device).type == "xpu")): kwargs["dtype"] = torch.float32 - return self.original_Tensor_to(return_xpu(device), *args, **kwargs) - else: - if not device_supports_fp64: - if kwargs.get("dtype", None) == torch.float64 and ((device is None and self.device.type == "xpu") or (device is not None and torch.device(device).type == "xpu")): - kwargs["dtype"] = torch.float32 - elif device == torch.float64 and self.device.type == "xpu": - device = torch.float32 - return self.original_Tensor_to(device, *args, **kwargs) + elif device == torch.float64 and self.device.type == "xpu": + device = torch.float32 + return self.original_Tensor_to(device, *args, **kwargs) + original_Tensor_cuda = torch.Tensor.cuda @wraps(torch.Tensor.cuda) @@ -277,6 +175,7 @@ def Tensor_cuda(self, device=None, *args, **kwargs): else: return original_Tensor_cuda(self, device, *args, **kwargs) + original_Tensor_pin_memory = torch.Tensor.pin_memory @wraps(torch.Tensor.pin_memory) def Tensor_pin_memory(self, device=None, *args, **kwargs): @@ -285,6 +184,7 @@ def Tensor_pin_memory(self, device=None, *args, **kwargs): else: return original_Tensor_pin_memory(self, device, *args, **kwargs) + original_UntypedStorage_init = torch.UntypedStorage.__init__ @wraps(torch.UntypedStorage.__init__) def UntypedStorage_init(*args, device=None, **kwargs): @@ -293,6 +193,7 @@ def UntypedStorage_init(*args, device=None, **kwargs): else: return original_UntypedStorage_init(*args, device=device, **kwargs) + if torch_version[0] > 2 or (torch_version[0] == 2 and torch_version[1] >= 4): original_UntypedStorage_to = torch.UntypedStorage.to @wraps(torch.UntypedStorage.to) @@ -310,6 +211,7 @@ if torch_version[0] > 2 or (torch_version[0] == 2 and torch_version[1] >= 4): else: return original_UntypedStorage_cuda(self, device=device, non_blocking=non_blocking, **kwargs) + original_torch_empty = torch.empty @wraps(torch.empty) def torch_empty(*args, device=None, **kwargs): @@ -318,15 +220,17 @@ def torch_empty(*args, device=None, **kwargs): else: return original_torch_empty(*args, device=device, **kwargs) + original_torch_randn = torch.randn @wraps(torch.randn) def torch_randn(*args, device=None, dtype=None, **kwargs): if dtype is bytes: dtype = None if check_cuda(device): - return original_torch_randn(*args, device=return_xpu(device), **kwargs) + return original_torch_randn(*args, device=return_xpu(device), dtype=dtype, **kwargs) else: - return original_torch_randn(*args, device=device, **kwargs) + return original_torch_randn(*args, device=device, dtype=dtype, **kwargs) + original_torch_ones = torch.ones @wraps(torch.ones) @@ -336,6 +240,7 @@ def torch_ones(*args, device=None, **kwargs): else: return original_torch_ones(*args, device=device, **kwargs) + original_torch_zeros = torch.zeros @wraps(torch.zeros) def torch_zeros(*args, device=None, **kwargs): @@ -344,6 +249,7 @@ def torch_zeros(*args, device=None, **kwargs): else: return original_torch_zeros(*args, device=device, **kwargs) + original_torch_full = torch.full @wraps(torch.full) def torch_full(*args, device=None, **kwargs): @@ -352,6 +258,7 @@ def torch_full(*args, device=None, **kwargs): else: return original_torch_full(*args, device=device, **kwargs) + original_torch_linspace = torch.linspace @wraps(torch.linspace) def torch_linspace(*args, device=None, **kwargs): @@ -360,6 +267,7 @@ def torch_linspace(*args, device=None, **kwargs): else: return original_torch_linspace(*args, device=device, **kwargs) + original_torch_eye = torch.eye @wraps(torch.eye) def torch_eye(*args, device=None, **kwargs): @@ -368,6 +276,7 @@ def torch_eye(*args, device=None, **kwargs): else: return original_torch_eye(*args, device=device, **kwargs) + original_torch_load = torch.load @wraps(torch.load) def torch_load(f, map_location=None, *args, **kwargs): @@ -376,6 +285,7 @@ def torch_load(f, map_location=None, *args, **kwargs): else: return original_torch_load(f, *args, map_location=map_location, **kwargs) + @wraps(torch.cuda.synchronize) def torch_cuda_synchronize(device=None): if check_cuda(device): @@ -383,6 +293,7 @@ def torch_cuda_synchronize(device=None): else: return torch.xpu.synchronize(device) + @wraps(torch.cuda.device) def torch_cuda_device(device): if check_cuda(device): @@ -390,6 +301,7 @@ def torch_cuda_device(device): else: return torch.xpu.device(device) + @wraps(torch.cuda.set_device) def torch_cuda_set_device(device): if check_cuda(device): @@ -397,6 +309,7 @@ def torch_cuda_set_device(device): else: torch.xpu.set_device(device) + # torch.Generator has to be a class for isinstance checks original_torch_Generator = torch.Generator class torch_Generator(original_torch_Generator): @@ -434,24 +347,14 @@ def ipex_hijacks(): torch.Generator = torch_Generator torch._C.Generator = torch_Generator - torch.backends.cuda.sdp_kernel = return_null_context - torch.nn.DataParallel = DummyDataParallel torch.UntypedStorage.is_cuda = is_cuda torch.amp.autocast_mode.autocast.__init__ = autocast_init torch.nn.functional.interpolate = interpolate - torch.nn.functional.scaled_dot_product_attention = scaled_dot_product_attention - torch.nn.functional.group_norm = functional_group_norm - torch.nn.functional.layer_norm = functional_layer_norm - torch.nn.functional.linear = functional_linear - torch.nn.functional.conv1d = functional_conv1d - torch.nn.functional.conv2d = functional_conv2d - torch.nn.functional.conv3d = functional_conv3d torch.nn.functional.pad = functional_pad - - torch.bmm = torch_bmm torch.fft.fftn = fft_fftn torch.fft.ifftn = fft_ifftn + if not device_supports_fp64: torch.from_numpy = from_numpy torch.as_tensor = as_tensor @@ -462,6 +365,18 @@ def ipex_hijacks(): except Exception: pass + if os.environ.get("IPEX_FORCE_ATTENTION_SLICE", "0") == "0": + if torch_version[0] > 2 or (torch_version[0] == 2 and torch_version[1] >= 7): + use_dynamic_attention = False # torch 2.7 has flash atten support + else: + use_dynamic_attention = True + else: + use_dynamic_attention = bool(os.environ.get("IPEX_FORCE_ATTENTION_SLICE", "0") == "1") + + if use_dynamic_attention: + from .attention import dynamic_scaled_dot_product_attention + torch.nn.functional.scaled_dot_product_attention = dynamic_scaled_dot_product_attention + # AMP: torch.amp.grad_scaler.GradScaler.__init__ = GradScaler_init torch.is_autocast_enabled = torch_is_autocast_enabled diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 578c96093..0821d5c4f 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -13,7 +13,7 @@ from modules import shared, devices, errors, model_quant, sd_models, sd_models_c processor = None model = None loaded: str = None -quant_args = {} +quant_args = None vlm_default = "Alibaba Qwen 2.5 VL 4B" vlm_models = { "Google Gemma 3 4B": "google/gemma-3-4b-it", @@ -27,12 +27,18 @@ vlm_models = { "Apple FastVLM 0.5B": "apple/FastVLM-0.5B", "Apple FastVLM 1.5B": "apple/FastVLM-1.5B", "Apple FastVLM 7B": "apple/FastVLM-7B", - "Microsoft Florence 2 Base": "microsoft/Florence-2-base-ft", # 0.5GB - "Microsoft Florence 2 Large": "microsoft/Florence-2-large-ft", # 1.5GB - "MiaoshouAI PromptGen 1.5 Base": "MiaoshouAI/Florence-2-base-PromptGen-v1.5@c06a5f02cc6071a5d65ee5d294cf3732d3097540", # 1.1GB - "MiaoshouAI PromptGen 1.5 Large": "MiaoshouAI/Florence-2-large-PromptGen-v1.5@28a42440e39c9c32b83f7ae74ec2b3d1540404f0", # 3.3GB - "MiaoshouAI PromptGen 2.0 Base": "MiaoshouAI/Florence-2-base-PromptGen-v2.0", # 1.1GB - "MiaoshouAI PromptGen 2.0 Large": "MiaoshouAI/Florence-2-large-PromptGen-v2.0", # 3.3GB + # "Microsoft Florence 2 Base": "microsoft/Florence-2-base-ft", # 0.5GB + # "Microsoft Florence 2 Large": "microsoft/Florence-2-large-ft", # 1.5GB + "Microsoft Florence 2 Base": "florence-community/Florence-2-base-ft", # 0.5GB + "Microsoft Florence 2 Large": "florence-community/Florence-2-large-ft", # 1.5GB + #"MiaoshouAI PromptGen 1.5 Base": "MiaoshouAI/Florence-2-base-PromptGen-v1.5@c06a5f02cc6071a5d65ee5d294cf3732d3097540", # 1.1GB + #"MiaoshouAI PromptGen 1.5 Large": "MiaoshouAI/Florence-2-large-PromptGen-v1.5@28a42440e39c9c32b83f7ae74ec2b3d1540404f0", # 3.3GB + #"MiaoshouAI PromptGen 2.0 Base": "MiaoshouAI/Florence-2-base-PromptGen-v2.0", # 1.1GB + #"MiaoshouAI PromptGen 2.0 Large": "MiaoshouAI/Florence-2-large-PromptGen-v2.0", # 3.3GB + "MiaoshouAI PromptGen 1.5 Base": "Disty0/Florence-2-base-PromptGen-v1.5", # 0.5GB + "MiaoshouAI PromptGen 1.5 Large": "Disty0/Florence-2-large-PromptGen-v1.5", # 1.5GB + "MiaoshouAI PromptGen 2.0 Base": "Disty0/Florence-2-base-PromptGen-v2.0", # 0.5GB + "MiaoshouAI PromptGen 2.0 Large": "Disty0/Florence-2-large-PromptGen-v2.0", # 1.5GB "CogFlorence 2.0 Large": "thwri/CogFlorence-2-Large-Freeze", # 1.6GB "CogFlorence 2.2 Large": "thwri/CogFlorence-2.2-Large", # 1.6GB "Moondream 2": "vikhyatk/moondream2", # 3.7GB @@ -88,14 +94,19 @@ def b64(image): def clean(response, question): strip = ['---', '\r', '\t', '**', '"', '“', '”', 'Assistant:', 'Caption:', '<|im_end|>', ''] - if isinstance(response, dict): - if 'task' in response: - response = response['task'] + if isinstance(response, str): + response = response.strip() + elif isinstance(response, dict): if 'answer' in response: response = response['answer'] - response = json.dumps(response) - if isinstance(response, list): + elif 'task' in response: + response = response['task'] + else: + response = json.dumps(response) + elif isinstance(response, list): response = response[0] + else: + response = str(response) question = question.replace('<', '').replace('>', '').replace('_', ' ') if question in response: response = response.split(question, 1)[1] @@ -354,7 +365,6 @@ def smol(question: str, image: Image.Image, repo: str = None, system_prompt: str repo, cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, - _attn_implementation="eager", **quant_args, ) processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) @@ -524,11 +534,13 @@ def moondream(question: str, image: Image.Image, repo: str = None): def florence(question: str, image: Image.Image, repo: str = None, revision: str = None): global processor, model, loaded # pylint: disable=global-statement _get_imports = transformers.dynamic_module_utils.get_imports + def get_imports(f): R = _get_imports(f) if "flash_attn" in R: R.remove("flash_attn") # flash_attn is optional return R + revision = None if '@' in repo: repo, revision = repo.split('@') @@ -536,6 +548,7 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str shared.log.debug(f'Interrogate load: vlm="{repo}" path="{shared.opts.hfcache_dir}"') transformers.dynamic_module_utils.get_imports = get_imports model = None + """ model = transformers.AutoModelForCausalLM.from_pretrained( repo, trust_remote_code=True, @@ -544,6 +557,15 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str cache_dir=shared.opts.hfcache_dir, **quant_args, ) + """ + model = transformers.Florence2ForConditionalGeneration.from_pretrained( + repo, + dtype=torch.bfloat16, + revision=revision, + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + **quant_args, + ) processor = transformers.AutoProcessor.from_pretrained(repo, trust_remote_code=True, revision=revision, cache_dir=shared.opts.hfcache_dir) transformers.dynamic_module_utils.get_imports = _get_imports loaded = repo @@ -607,25 +629,28 @@ def interrogate(question:str='', system_prompt:str=None, prompt:str=None, image: global quant_args # pylint: disable=global-statement jobid = shared.state.begin('Interrogate LLM') t0 = time.time() - quant_args = model_quant.create_config(module='LLM') + if quant_args is None: + quant_args = model_quant.create_config(module='LLM') model_name = model_name or shared.opts.interrogate_vlm_model if isinstance(image, list): image = image[0] if len(image) > 0 else None if isinstance(image, dict) and 'name' in image: image = Image.open(image['name']) - if image is None: - return '' - if image.width > 768 or image.height > 768: - image.thumbnail((768, 768), Image.Resampling.LANCZOS) - if image.mode != 'RGB': - image = image.convert('RGB') + if isinstance(image, Image.Image): + if image.width > 768 or image.height > 768: + image.thumbnail((768, 768), Image.Resampling.LANCZOS) + if image.mode != 'RGB': + image = image.convert('RGB') if prompt is not None and len(prompt) > 0: question = prompt if len(question) < 2: question = "Describe the image." + + """ if shared.sd_loaded: from modules.sd_models import apply_balanced_offload # prevent circular import apply_balanced_offload(shared.sd_model) + """ from modules import modelloader modelloader.hf_login() @@ -638,9 +663,9 @@ def interrogate(question:str='', system_prompt:str=None, prompt:str=None, image: if vqa_model is None: shared.log.error(f'Interrogate: type=vlm model="{model_name}" unknown') return '' - if image is None: - shared.log.error(f'Interrogate: type=vlm model="{model_name}" no input image') - return '' + # if image is None: + # shared.log.error(f'Interrogate: type=vlm model="{model_name}" no input image') + # return '' if 'git' in vqa_model.lower(): answer = git(question, image, vqa_model) diff --git a/modules/ipadapter.py b/modules/ipadapter.py index 676045d1f..88a01e2cd 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -354,6 +354,18 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt adapter_scales[i] = 0.00 pipe.set_ip_adapter_scale(adapter_scales if len(adapter_scales) > 1 else adapter_scales[0]) ip_str = [f'{os.path.splitext(adapter)[0]}:{scale}:{start}:{end}:{crop}' for adapter, scale, start, end, crop in zip(adapter_names, adapter_scales, adapter_starts, adapter_ends, adapter_crops)] + if hasattr(pipe, 'transformer') and 'Nunchaku' in pipe.transformer.__class__.__name__: + if isinstance(repos, str): + sd_models.clear_caches(full=True) + import accelerate + accelerate.hooks.remove_hook_from_module(pipe.transformer, recurse=True) + pipe.transformer = pipe.transformer.to(devices.device) + from nunchaku.models.ip_adapter.diffusers_adapters import apply_IPA_on_pipe + apply_IPA_on_pipe(pipe, ip_adapter_scale=adapter_scales[0], repo_id=repos) + pipe = sd_models.apply_balanced_offload(pipe) + shared.log.debug(f'IP adapter load: engine=nunchaku scale={adapter_scales[0]} repo="{repos}"') + else: + shared.log.error('IP adapter: Nunchaku only supports single adapter') p.task_args['ip_adapter_image'] = crop_images(adapter_images, adapter_crops) if len(adapter_masks) > 0: p.cross_attention_kwargs = { 'ip_adapter_masks': adapter_masks } diff --git a/modules/loader.py b/modules/loader.py index efd58ff11..a224c05c7 100644 --- a/modules/loader.py +++ b/modules/loader.py @@ -51,10 +51,15 @@ if torch.__version__.startswith('2.5.0'): errors.log.warning(f'Disabling cuDNN for SDP on torch={torch.__version__}') torch.backends.cuda.enable_cudnn_sdp(False) try: - import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import + import intel_extension_for_pytorch as ipex # pylint: disable=import-error,unused-import errors.log.debug(f'Load IPEX=={ipex.__version__}') except Exception: pass +try: + import torch.distributed.distributed_c10d as _c10d # pylint: disable=unused-import,ungrouped-imports +except Exception: + errors.log.warning('Loader: torch is not built with distributed support') + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) warnings.filterwarnings(action="ignore", category=UserWarning, module="torchvision") @@ -123,15 +128,20 @@ timer.startup.record("pydantic") import tqdm as tqdm_lib # pylint: disable=C0411 from tqdm.rich import tqdm # pylint: disable=W0611,C0411 -import diffusers.utils.import_utils # pylint: disable=W0611,C0411 -diffusers.utils.import_utils._k_diffusion_available = True # pylint: disable=protected-access # monkey-patch since we use k-diffusion from git -diffusers.utils.import_utils._k_diffusion_version = '0.0.12' # pylint: disable=protected-access +try: + import diffusers.utils.import_utils # pylint: disable=W0611,C0411 + diffusers.utils.import_utils._k_diffusion_available = True # pylint: disable=protected-access # monkey-patch since we use k-diffusion from git + diffusers.utils.import_utils._k_diffusion_version = '0.0.12' # pylint: disable=protected-access -import diffusers # pylint: disable=W0611,C0411 -import diffusers.loaders.single_file # pylint: disable=W0611,C0411 -diffusers.loaders.single_file.logging.tqdm = partial(tqdm, unit='C') -logging.getLogger("diffusers.loaders.single_file").setLevel(logging.ERROR) -timer.startup.record("diffusers") + import diffusers # pylint: disable=W0611,C0411 + import diffusers.loaders.single_file # pylint: disable=W0611,C0411 + diffusers.loaders.single_file.logging.tqdm = partial(tqdm, unit='C') + logging.getLogger("diffusers.loaders.single_file").setLevel(logging.ERROR) + timer.startup.record("diffusers") +except Exception as e: + errors.log.error(f'Loader: diffusers=={diffusers.__version__ if "diffusers" in sys.modules else None} {e}') + errors.log.error('Please restart re-run the installer') + sys.exit(1) import huggingface_hub # pylint: disable=W0611,C0411 timer.startup.record("hfhub") diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index d18444ea0..ee657d2d1 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -2,7 +2,7 @@ from typing import List import os import re import numpy as np -from modules.lora import networks, lora_overrides, lora_load +from modules.lora import networks, lora_overrides, lora_load, lora_diffusers from modules.lora import lora_common as l from modules import extra_networks, shared, sd_models @@ -83,21 +83,34 @@ def infotext(p): p.extra_generation_params["LoRA hashes"] = ", ".join(network_hashes) +def to_float(value): + try: + return float(value) + except (ValueError, TypeError): + return value + + def parse(p, params_list, step=0): names = [] te_multipliers = [] unet_multipliers = [] dyn_dims = [] + lora_modules = [] for params in params_list: - assert params.items names.append(params.positional[0]) - te_multiplier = params.named.get("te", params.positional[1] if len(params.positional) > 1 else shared.opts.extra_networks_default_multiplier) + + default_multiplier = params.positional[1] if len(params.positional) > 1 else shared.opts.extra_networks_default_multiplier + default_multiplier = to_float(default_multiplier) + if isinstance(default_multiplier, str) and "@" not in default_multiplier: + default_multiplier = shared.opts.extra_networks_default_multiplier + + te_multiplier = params.named.get("te", default_multiplier) if isinstance(te_multiplier, str) and "@" in te_multiplier: te_multiplier = get_stepwise(te_multiplier, step, p.steps) else: - te_multiplier = float(te_multiplier) - unet_multiplier = [params.positional[2] if len(params.positional) > 2 else te_multiplier] * 3 - unet_multiplier = [params.named.get("unet", unet_multiplier[0])] * 3 + te_multiplier = to_float(te_multiplier) + + unet_multiplier = 3 * [params.named.get("unet", te_multiplier)] # fill all 3 with same value unet_multiplier[0] = params.named.get("in", unet_multiplier[0]) unet_multiplier[1] = params.named.get("mid", unet_multiplier[1]) unet_multiplier[2] = params.named.get("out", unet_multiplier[2]) @@ -105,13 +118,23 @@ def parse(p, params_list, step=0): if isinstance(unet_multiplier[i], str) and "@" in unet_multiplier[i]: unet_multiplier[i] = get_stepwise(unet_multiplier[i], step, p.steps) else: - unet_multiplier[i] = float(unet_multiplier[i]) - dyn_dim = int(params.positional[3]) if len(params.positional) > 3 else None - dyn_dim = int(params.named["dyn"]) if "dyn" in params.named else dyn_dim + unet_multiplier[i] = to_float(unet_multiplier[i]) + + dyn_dim = int(params.named["dyn"]) if "dyn" in params.named else None te_multipliers.append(te_multiplier) unet_multipliers.append(unet_multiplier) dyn_dims.append(dyn_dim) - return names, te_multipliers, unet_multipliers, dyn_dims + + lora_module = [] + if 'high' in params.positional or 'HIGH 14B' in params.positional[0]: + lora_module.append('transformer') + if 'low' in params.positional or 'LOW 14B' in params.positional[0]: + lora_module.append('transformer_2') + if params.named.get('module', None) is not None: + lora_module.append(params.named['module'].lower()) + lora_modules.append(lora_module) + + return names, te_multipliers, unet_multipliers, dyn_dims, lora_modules def unload_diffusers(): @@ -168,7 +191,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): if len(params_list) > 0 and not self.active: # activate patches once self.active = True self.model = shared.opts.sd_model_checkpoint - names, te_multipliers, unet_multipliers, dyn_dims = parse(p, params_list, step) + names, te_multipliers, unet_multipliers, dyn_dims, lora_modules = parse(p, params_list, step) requested = self.signature(names, te_multipliers, unet_multipliers) load_method = lora_overrides.get_method() @@ -181,7 +204,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): has_changed = False # diffusers handles its own loading if len(exclude) == 0: jobid = shared.state.begin('LoRA') - lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load only on first call + lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims, lora_modules) # load only on first call sd_models.set_diffuser_offload(shared.sd_model, op="model") shared.state.end(jobid) elif load_method == 'nunchaku': @@ -208,7 +231,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): shared.log.info(f'Network load: type=LoRA apply={[n.name for n in l.loaded_networks]} method={load_method} mode={"fuse" if shared.opts.lora_fuse_diffusers else "backup"} te={te_multipliers} unet={unet_multipliers} time={l.timer.summary}') def deactivate(self, p): - if len(lora_load.diffuser_loaded) > 0: + if len(lora_diffusers.diffuser_loaded) > 0: if not (shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled is True): unload_diffusers() if self.active and l.debug: diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index 0343d7b14..f18922040 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -47,6 +47,17 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n self.network_weights_backup = weight.clone().to(devices.cpu) if hasattr(self, "sdnq_dequantizer"): self.sdnq_dequantizer_backup = self.sdnq_dequantizer.to(devices.cpu) + self.sdnq_scale_backup = self.scale.clone().to(devices.cpu) + if self.zero_point is not None: + self.sdnq_zero_point_backup = self.zero_point.clone().to(devices.cpu) + else: + self.sdnq_zero_point_backup = None + if self.svd_up is not None: + self.sdnq_svd_up_backup = self.svd_up.clone().to(devices.cpu) + self.sdnq_svd_down_backup = self.svd_down.clone().to(devices.cpu) + else: + self.sdnq_svd_up_backup = None + self.sdnq_svd_down_backup = None if bias_backup is None: if getattr(self, 'bias', None) is not None: @@ -80,9 +91,23 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. try: t0 = time.time() if hasattr(self, "sdnq_dequantizer_backup"): - weight = self.sdnq_dequantizer_backup.to(devices.device)(self.weight.to(devices.device), skip_quantized_matmul=self.sdnq_dequantizer_backup.use_quantized_matmul) + weight = self.sdnq_dequantizer_backup.to(devices.device)( + self.weight.to(devices.device), + self.sdnq_scale_backup.to(devices.device), + self.sdnq_zero_point_backup.to(devices.device) if self.sdnq_zero_point_backup is not None else None, + self.sdnq_svd_up_backup.to(devices.device) if self.sdnq_svd_up_backup is not None else None, + self.sdnq_svd_down_backup.to(devices.device) if self.sdnq_svd_down_backup is not None else None, + skip_quantized_matmul=self.sdnq_dequantizer_backup.use_quantized_matmul + ) elif hasattr(self, "sdnq_dequantizer"): - weight = self.sdnq_dequantizer.to(devices.device)(self.weight.to(devices.device), skip_quantized_matmul=self.sdnq_dequantizer.use_quantized_matmul) + weight = self.sdnq_dequantizer.to(devices.device)( + self.weight.to(devices.device), + self.scale.to(devices.device), + self.zero_point.to(devices.device) if self.zero_point is not None else None, + self.svd_up.to(devices.device) if self.svd_up is not None else None, + self.svd_down.to(devices.device) if self.svd_down is not None else None, + skip_quantized_matmul=self.sdnq_dequantizer.use_quantized_matmul + ) else: weight = self.weight.to(devices.device) # must perform calc on gpu due to performance updown, ex_bias = module.calc_updown(weight) @@ -142,18 +167,37 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G try: from modules.sdnq import sdnq_quantize_layer if hasattr(self, "sdnq_dequantizer_backup"): - sdnq_dequantizer = self.sdnq_dequantizer_backup.to(devices.device) + weights_dtype = self.sdnq_dequantizer_backup.weights_dtype + dequant_weight = self.sdnq_dequantizer_backup.to(devices.device)( + model_weights.to(devices.device), + self.sdnq_scale_backup.to(devices.device), + self.sdnq_zero_point_backup.to(devices.device) if self.sdnq_zero_point_backup is not None else None, + self.sdnq_svd_up_backup.to(devices.device) if self.sdnq_svd_up_backup is not None else None, + self.sdnq_svd_down_backup.to(devices.device) if self.sdnq_svd_down_backup is not None else None, + skip_quantized_matmul=self.sdnq_dequantizer_backup.use_quantized_matmul + ) else: - sdnq_dequantizer = self.sdnq_dequantizer.to(devices.device) - dequant_weight = sdnq_dequantizer(model_weights.to(devices.device), skip_quantized_matmul=sdnq_dequantizer.use_quantized_matmul) + weights_dtype = self.sdnq_dequantizer.weights_dtype + dequant_weight = self.sdnq_dequantizer.to(devices.device)( + model_weights.to(devices.device), + self.scale.to(devices.device), + self.zero_point.to(devices.device) if self.zero_point is not None else None, + self.svd_up.to(devices.device) if self.svd_up is not None else None, + self.svd_down.to(devices.device) if self.svd_down is not None else None, + skip_quantized_matmul=self.sdnq_dequantizer.use_quantized_matmul + ) + new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32) self.weight = torch.nn.Parameter(new_weight, requires_grad=False) - self.sdnq_dequantizer = None + del self.sdnq_dequantizer, self.scale, self.zero_point, self.svd_up, self.svd_down self = sdnq_quantize_layer( self, - sdnq_dequantizer.weights_dtype, + weights_dtype=weights_dtype, torch_dtype=devices.dtype, group_size=shared.opts.sdnq_quantize_weights_group_size, + svd_rank=shared.opts.sdnq_svd_rank, + svd_steps=shared.opts.sdnq_svd_steps, + use_svd=shared.opts.sdnq_use_svd, quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, @@ -231,7 +275,17 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn self.weight = torch.nn.Parameter(weights_backup.to(device), requires_grad=False) if hasattr(self, "sdnq_dequantizer_backup"): self.sdnq_dequantizer = self.sdnq_dequantizer_backup.to(device) - del self.sdnq_dequantizer_backup + self.scale = torch.nn.Parameter(self.sdnq_scale_backup.to(device), requires_grad=False) + if self.sdnq_zero_point_backup is not None: + self.zero_point = torch.nn.Parameter(self.sdnq_zero_point_backup.to(device), requires_grad=False) + else: + self.zero_point = None + if self.sdnq_svd_up_backup is not None: + self.svd_up = torch.nn.Parameter(self.sdnq_svd_up_backup.to(device), requires_grad=False) + self.svd_down = torch.nn.Parameter(self.sdnq_svd_down_backup.to(device), requires_grad=False) + else: + self.svd_up, self.svd_down = None, None + del self.sdnq_dequantizer_backup, self.sdnq_scale_backup, self.sdnq_zero_point_backup, self.sdnq_svd_up_backup, self.sdnq_svd_down_backup if bias_backup is not None: self.bias = None diff --git a/modules/lora/lora_diffusers.py b/modules/lora/lora_diffusers.py new file mode 100644 index 000000000..29402febe --- /dev/null +++ b/modules/lora/lora_diffusers.py @@ -0,0 +1,92 @@ +from typing import Union +import os +import time +import diffusers +from modules import shared, errors +from modules.lora import network +from modules.lora import lora_common as l + + +diffuser_loaded = [] +diffuser_scales = [] + + +def load_per_module(sd_model: diffusers.DiffusionPipeline, filename: str, adapter_name: str, lora_modules: list[str]): + shared.log.debug(f'LoRA load: modules={lora_modules}') + try: + state_dict = sd_model.lora_state_dict(filename) + if isinstance(state_dict, tuple) and len(state_dict) == 2: + state_dict, network_alphas = state_dict + else: + network_alphas = {} + except Exception as e: + shared.log.error(f'LoRA load: {e}') + if l.debug: + errors.display(e, "LoRA") + return None + for lora_module in lora_modules: + if lora_module == 'transformer': + if hasattr(sd_model, 'transformer') and sd_model.transformer is not None: + sd_model.load_lora_into_transformer(state_dict, transformer=sd_model.transformer, adapter_name=adapter_name) + else: + shared.log.warning(f'LoRA load: requested={lora_module} missing') + elif lora_module == 'transformer_2': + if hasattr(sd_model, 'transformer_2') and sd_model.transformer_2 is not None: + sd_model.load_lora_into_transformer(state_dict, transformer=sd_model.transformer_2, adapter_name=adapter_name) + else: + shared.log.warning(f'LoRA load: requested={lora_module} missing') + elif lora_module == 'unet': + if hasattr(sd_model, 'unet') and sd_model.unet is not None: + sd_model.load_lora_into_unet(state_dict, network_alphas, unet=sd_model.unet, adapter_name=adapter_name) + else: + shared.log.warning(f'LoRA load: requested={lora_module} missing') + elif lora_module == 'text_encoder' or lora_module == 'te': + if hasattr(sd_model, 'text_encoder') and sd_model.text_encoder is not None: + sd_model.load_lora_into_text_encoder(state_dict, network_alphas, text_encoder=sd_model.text_encoder, adapter_name=adapter_name) + else: + shared.log.warning(f'LoRA load: requested={lora_module} missing') + else: + shared.log.warning(f'LoRA load: requested={lora_module} unknown') + return adapter_name + + +def load_diffusers(name: str, network_on_disk: network.NetworkOnDisk, lora_scale:float=shared.opts.extra_networks_default_multiplier, lora_module=None) -> Union[network.Network, None]: + t0 = time.time() + name = name.replace(".", "_") + sd_model: diffusers.DiffusionPipeline = getattr(shared.sd_model, "pipe", shared.sd_model) + shared.log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_diffusers}') + if not hasattr(sd_model, 'load_lora_weights'): + shared.log.error(f'Network load: type=LoRA class={sd_model.__class__} does not implement load lora') + return None + try: + if lora_module is not None and isinstance(lora_module, list) and len(lora_module) > 0: + name = load_per_module(sd_model, network_on_disk.filename, adapter_name=name, lora_modules=lora_module) + else: + sd_model.load_lora_weights(network_on_disk.filename, adapter_name=name) + except Exception as e: + if 'already in use' in str(e): + pass + else: + if 'following keys have not been correctly renamed' in str(e): + shared.log.error(f'Network load: type=LoRA name="{name}" diffusers unsupported format') + elif 'object has no attribute' in str(e): + shared.log.error(f'Network load: type=LoRA name="{name}" diffusers empty module') + else: + shared.log.error(f'Network load: type=LoRA name="{name}" {e}') + if l.debug: + errors.display(e, "LoRA") + return None + if name is None: + return None + if name not in diffuser_loaded: + list_adapters = sd_model.get_list_adapters() + list_adapters = [adapter for adapters in list_adapters.values() for adapter in adapters] + if name not in list_adapters: + shared.log.error(f'Network load: type=LoRA name="{name}" adapters={list_adapters} not loaded') + else: + diffuser_loaded.append(name) + diffuser_scales.append(lora_scale) + net = network.Network(name, network_on_disk) + net.mtime = os.path.getmtime(network_on_disk.filename) + l.timer.activate += time.time() - t0 + return net diff --git a/modules/lora/lora_load.py b/modules/lora/lora_load.py index 0b74c6c6d..f629c2772 100644 --- a/modules/lora/lora_load.py +++ b/modules/lora/lora_load.py @@ -3,12 +3,10 @@ import os import time import concurrent from modules import shared, errors, sd_models, sd_models_compile, files_cache -from modules.lora import network, lora_overrides, lora_convert +from modules.lora import network, lora_overrides, lora_convert, lora_diffusers from modules.lora import lora_common as l -diffuser_loaded = [] -diffuser_scales = [] lora_cache = {} available_networks = {} available_network_aliases = {} @@ -17,41 +15,6 @@ available_network_hash_lookup = {} dump_lora_keys = os.environ.get('SD_LORA_DUMP', None) is not None -def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_default_multiplier) -> Union[network.Network, None]: - t0 = time.time() - name = name.replace(".", "_") - sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) - shared.log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_diffusers}') - if not hasattr(sd_model, 'load_lora_weights'): - shared.log.error(f'Network load: type=LoRA class={sd_model.__class__} does not implement load lora') - return None - try: - sd_model.load_lora_weights(network_on_disk.filename, adapter_name=name) - except Exception as e: - if 'already in use' in str(e): - pass - else: - if 'The following keys have not been correctly renamed' in str(e): - shared.log.error(f'Network load: type=LoRA name="{name}" diffusers unsupported format') - else: - shared.log.error(f'Network load: type=LoRA name="{name}" {e}') - if l.debug: - errors.display(e, "LoRA") - return None - if name not in diffuser_loaded: - list_adapters = sd_model.get_list_adapters() - list_adapters = [adapter for adapters in list_adapters.values() for adapter in adapters] - if name not in list_adapters: - shared.log.error(f'Network load: type=LoRA name="{name}" adapters={list_adapters} not loaded') - else: - diffuser_loaded.append(name) - diffuser_scales.append(lora_scale) - net = network.Network(name, network_on_disk) - net.mtime = os.path.getmtime(network_on_disk.filename) - l.timer.activate += time.time() - t0 - return net - - def lora_dump(lora, dct): import tempfile sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) @@ -73,7 +36,7 @@ def lora_dump(lora, dct): f.write(line + "\n") -def load_safetensors(name, network_on_disk) -> Union[network.Network, None]: +def load_safetensors(name, network_on_disk: network.NetworkOnDisk) -> Union[network.Network, None]: if not shared.sd_loaded: return None @@ -261,15 +224,15 @@ def gather_networks(names): return networks_on_disk -def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None): +def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None, lora_modules=None): networks_on_disk = gather_networks(names) failed_to_load_networks = [] recompile_model, skip_lora_load = maybe_recompile_model(names, te_multipliers) sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) l.loaded_networks.clear() - diffuser_loaded.clear() - diffuser_scales.clear() + lora_diffusers.diffuser_loaded.clear() + lora_diffusers.diffuser_scales.clear() t0 = time.time() for i, (network_on_disk, name) in enumerate(zip(networks_on_disk, names)): @@ -279,11 +242,13 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non if l.debug: shared.log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" hash="{shorthash}"') try: + lora_scale = te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier + lora_module = lora_modules[i] if lora_modules and len(lora_modules) > i else None if recompile_model: - shared.compiled_model_state.lora_model.append(f"{name}:{te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier}") + shared.compiled_model_state.lora_model.append(f"{name}:{lora_scale}") lora_method = lora_overrides.get_method(shorthash) if lora_method == 'diffusers': - net = load_diffusers(name, network_on_disk, lora_scale=te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier) + net = lora_diffusers.load_diffusers(name, network_on_disk, lora_scale, lora_module) elif lora_method == 'nunchaku': pass # handled directly from extra_networks_lora.load_nunchaku else: @@ -311,21 +276,21 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non name = next(iter(lora_cache)) lora_cache.pop(name, None) - if not skip_lora_load and len(diffuser_loaded) > 0: - shared.log.debug(f'Network load: type=LoRA loaded={diffuser_loaded} available={sd_model.get_list_adapters()} active={sd_model.get_active_adapters()} scales={diffuser_scales}') + if not skip_lora_load and len(lora_diffusers.diffuser_loaded) > 0: + shared.log.debug(f'Network load: type=LoRA loaded={lora_diffusers.diffuser_loaded} available={sd_model.get_list_adapters()} active={sd_model.get_active_adapters()} scales={lora_diffusers.diffuser_scales}') try: t1 = time.time() if l.debug: shared.log.trace(f'Network load: type=LoRA list={sd_model.get_list_adapters()}') shared.log.trace(f'Network load: type=LoRA active={sd_model.get_active_adapters()}') - sd_model.set_adapters(adapter_names=diffuser_loaded, adapter_weights=diffuser_scales) + sd_model.set_adapters(adapter_names=lora_diffusers.diffuser_loaded, adapter_weights=lora_diffusers.diffuser_scales) except Exception as e: shared.log.error(f'Network load: type=LoRA action=set {e}') if l.debug: errors.display(e, 'LoRA') try: if shared.opts.lora_fuse_diffusers and not lora_overrides.disable_fuse(): - sd_model.fuse_lora(adapter_names=diffuser_loaded, lora_scale=1.0, fuse_unet=True, fuse_text_encoder=True) # diffusers with fuse uses fixed scale since later apply does the scaling + sd_model.fuse_lora(adapter_names=lora_diffusers.diffuser_loaded, lora_scale=1.0, fuse_unet=True, fuse_text_encoder=True) # diffusers with fuse uses fixed scale since later apply does the scaling sd_model.unload_lora_weights() l.timer.activate += time.time() - t1 except Exception as e: diff --git a/modules/lora/lora_nunchaku.py b/modules/lora/lora_nunchaku.py index 63a71b46f..de4773158 100644 --- a/modules/lora/lora_nunchaku.py +++ b/modules/lora/lora_nunchaku.py @@ -15,6 +15,9 @@ def load_nunchaku(names, strengths): is_changed = loras != previously_loaded if not is_changed: return False + if not hasattr(shared.sd_model, 'transformer') or not hasattr(shared.sd_model.transformer, 'update_lora_params'): + shared.log.error(f'Network load: type=LoRA method=nunchaku model={shared.sd_model.__class__.__name__} unsupported') + return False previously_loaded = loras try: @@ -27,7 +30,7 @@ def load_nunchaku(names, strengths): lora_common.timer.load = t1 - t0 shared.log.debug(f"Network load: type=LoRA method=nunchaku loras={names} strength={strengths} time={t1-t0:.3f}") except Exception as e: - shared.log.errors(f'Network load: type=LoRA method=nunchaku {e}') + shared.log.error(f'Network load: type=LoRA method=nunchaku {e}') if lora_common.debug: errors.display(e, 'LoRA') return is_changed diff --git a/modules/lora/lora_overrides.py b/modules/lora/lora_overrides.py index 61f2dad46..5889e1dec 100644 --- a/modules/lora/lora_overrides.py +++ b/modules/lora/lora_overrides.py @@ -61,7 +61,9 @@ def get_method(shorthash=''): use_diffusers = use_diffusers or any(x.startswith(shorthash) for x in maybe_diffusers) if shared.opts.lora_force_diffusers and len(shorthash) > 4: use_diffusers = use_diffusers or any(x.startswith(shorthash) for x in force_diffusers) - use_nunchaku = hasattr(shared.sd_model, 'transformer') and 'Nunchaku' in shared.sd_model.transformer.__class__.__name__ + nunchaku_dit = hasattr(shared.sd_model, 'transformer') and 'Nunchaku' in shared.sd_model.transformer.__class__.__name__ + nunchaku_unet = hasattr(shared.sd_model, 'unet') and 'Nunchaku' in shared.sd_model.unet.__class__.__name__ + use_nunchaku = nunchaku_dit or nunchaku_unet if use_nunchaku: return 'nunchaku' elif use_diffusers: diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 79837f1fa..7d9e6ae84 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -13,7 +13,7 @@ from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsamp debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None -engine, model = 'LTX Video', 'LTXVideo 0.9.7 13B' +# engine, model = 'LTX Video', 'LTXVideo 0.9.7 13B' upsample_repo_id = "a-r-r-o-w/LTX-Video-0.9.7-Latent-Spatial-Upsampler-diffusers" upsample_pipe = None queue_lock = threading.Lock() @@ -21,6 +21,7 @@ queue_lock = threading.Lock() def run_ltx(task_id, _ui_state, + model:str, prompt:str, negative:str, styles:list[str], @@ -65,7 +66,10 @@ def run_ltx(task_id, progress.finish_task(task_id) yield None, f'LTX Error: {str(e)}' - from diffusers import LTXConditionPipeline # pylint: disable=unused-import + if model is None or len(model) == 0: + yield from abort('Video: no model selected', ok=True) + return + # from diffusers import LTXConditionPipeline # pylint: disable=unused-import check_av() progress.add_task_to_queue(task_id) with queue_lock: @@ -73,9 +77,14 @@ def run_ltx(task_id, memstats.reset_stats() timer.process.reset() yield None, 'LTX: Loading...' + engine = 'LTX Video' load_model(engine, model) + debug(f'Video: cls={shared.sd_model.__class__.__name__} op=init model="{model}"') + if not shared.sd_model.__class__.__name__.startswith("LTX"): + yield from abort(f'Video: cls={shared.sd_model.__class__.__name__} selected model is not LTX model', ok=True) + return - shared.state.begin('Video', task_id=task_id) + videojob = shared.state.begin('Video', task_id=task_id) shared.state.job_count = 1 p = processing.StableDiffusionProcessingVideo( @@ -105,7 +114,7 @@ def run_ltx(task_id, prompt, negative, networks = get_prompts(prompt, negative, styles) sampler_name = processing.get_sampler_name(sampler_index) sd_samplers.create_sampler(sampler_name, shared.sd_model) - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=init prompt="{prompt}" negative="{negative}" styles={styles} networks={networks} sampler={shared.sd_model.scheduler.__class__.__name__}') + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=init styles={styles} networks={networks} sampler={shared.sd_model.scheduler.__class__.__name__}') extra_networks.activate(p, networks) t0 = time.time() @@ -127,6 +136,7 @@ def run_ltx(task_id, if len(conditions) > 0: base_args["conditions"] = conditions yield None, 'LTX: Generate in progress...' + samplejob = shared.state.begin('Sample') try: latents = shared.sd_model(**base_args).frames[0] except AssertionError as e: @@ -141,10 +151,11 @@ def run_ltx(task_id, timer.process.add('offload', t1 - t0) timer.process.add('base', t2 - t1) timer.process.add('offload', t3 - t2) + shared.state.end(samplejob) if upsample_enable: t4 = time.time() - shared.state.begin('Upsample') + upsamplejob = shared.state.begin('Upsample') global upsample_pipe # pylint: disable=global-statement upsample_pipe = load_upsample(upsample_pipe, upsample_repo_id) upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) @@ -172,11 +183,11 @@ def run_ltx(task_id, t6 = time.time() timer.process.add('upsample', t5 - t4) timer.process.add('offload', t6 - t5) - shared.state.end() + shared.state.end(upsamplejob) if refine_enable: t7 = time.time() - shared.state.begin('Refine') + refinejob = shared.state.begin('Refine') shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) refine_args = { "prompt": prompt, @@ -211,7 +222,7 @@ def run_ltx(task_id, t9 = time.time() timer.process.add('refine', t8 - t7) timer.process.add('offload', t9 - t8) - shared.state.end() + shared.state.end(refinejob) extra_networks.deactivate(p) @@ -250,7 +261,7 @@ def run_ltx(task_id, fps = f'{num_frames/(t_end-t0):.2f}' its = f'{(steps)/(t_end-t0):.2f}' - shared.state.end() + shared.state.end(videojob) progress.finish_task(task_id) shared.log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={fps} its={its} resolution={resolution} time={t_end-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index 27b033af5..04cea1d1e 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -3,6 +3,7 @@ import gradio as gr from modules import shared, ui_sections, ui_symbols, ui_common from modules.ui_components import ToolButton from modules.video_models.video_utils import get_codecs +from modules.video_models.models_def import models from modules.ltx import ltx_process @@ -14,6 +15,9 @@ def create_ui(prompt, negative, styles, overrides): with gr.Column(variant='compact', elem_id="ltx_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): generate = gr.Button('Generate', elem_id="ltx_generate_btn", variant='primary', visible=False) + with gr.Row(): + ltx_models = [m.name for m in models['LTX Video']] + model = gr.Dropdown(label='LTX model', choices=ltx_models, value=ltx_models[0]) with gr.Accordion(open=True, label="LTX size", elem_id='ltx_generate_accordion'): with gr.Row(): width, height = ui_sections.create_resolution_inputs('ltx', default_width=832, default_height=480) @@ -73,6 +77,7 @@ def create_ui(prompt, negative, styles, overrides): state_inputs = [task_id, ui_state] video_inputs = [ + model, prompt, negative, styles, width, height, frames, steps, sampler_index, seed, diff --git a/modules/ltx/ltx_util.py b/modules/ltx/ltx_util.py index a07434734..ddc53a323 100644 --- a/modules/ltx/ltx_util.py +++ b/modules/ltx/ltx_util.py @@ -4,7 +4,12 @@ from PIL import Image from modules import devices, shared, sd_models, timer, extra_networks +loaded_model: str = None + + def get_bucket(size: int): + if not hasattr(shared.sd_model, 'vae_temporal_compression_ratio'): + return int(size) - (int(size) % 16) return int(size) - (int(size) % shared.sd_model.vae_temporal_compression_ratio) @@ -13,11 +18,19 @@ def get_frames(frames: int): def load_model(engine: str, model: str): + global loaded_model # pylint: disable=global-statement + if loaded_model == model: + return + if model is None or model == '' or model=='None': + loaded_model = None + shared.sd_model = None + return t0 = time.time() from modules.video_models import models_def, video_load selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] - shared.log.info(f'Video load: cls={selected.repo_cls.__name__} repo="{selected.repo}"') + shared.log.info(f'Video load: engine="{engine}" selected="{model}" {selected}') video_load.load_model(selected) + loaded_model = model t1 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) t2 = time.time() diff --git a/modules/memstats.py b/modules/memstats.py index fdc94df37..68a7d27ea 100644 --- a/modules/memstats.py +++ b/modules/memstats.py @@ -54,11 +54,28 @@ def ram_stats(): ram_total = 100 * res.rss / process.memory_percent() ram_total = min(ram_total, get_docker_limit(), get_runpod_limit()) ram['total'] = gb(ram_total) - ram['used'] = gb(res.rss) - ram['free'] = round(ram['total'] - ram['used']) + ram['rss'] = gb(res.rss) except Exception as e: ram['total'] = 0 + ram['rss'] = 0 + ram['error'] = str(e) + if not fail_once: + shared.log.error(f'RAM stats: {e}') + errors.display(e, 'RAM stats') + fail_once = True + try: + vmem = psutil.virtual_memory() + ram['used'] = gb(vmem.used) if hasattr(vmem, 'used') else 0 + ram['free'] = gb(vmem.free) if hasattr(vmem, 'free') else 0 + ram['avail'] = gb(vmem.available) if hasattr(vmem, 'available') else 0 + ram['buffers'] = gb(vmem.buffers) if hasattr(vmem, 'buffers') else 0 + ram['cached'] = gb(vmem.cached) if hasattr(vmem, 'cached') else 0 + except Exception as e: ram['used'] = 0 + ram['free'] = 0 + ram['avail'] = 0 + ram['buffers'] = 0 + ram['cached'] = 0 ram['error'] = str(e) if not fail_once: shared.log.error(f'RAM stats: {e}') diff --git a/modules/mit_nunchaku.py b/modules/mit_nunchaku.py index a06e7cfda..9817c4e53 100644 --- a/modules/mit_nunchaku.py +++ b/modules/mit_nunchaku.py @@ -4,7 +4,7 @@ from installer import log, pip from modules import devices -ver = '1.0.0' +ver = '1.0.1' ok = False @@ -15,7 +15,11 @@ def check(): try: import nunchaku import nunchaku.utils - log.info(f'Nunchaku: path={nunchaku.__path__} precision={nunchaku.utils.get_precision()}') + from nunchaku import __version__ + log.info(f'Nunchaku: path={nunchaku.__path__} version={__version__.__version__} precision={nunchaku.utils.get_precision()}') + if __version__.__version__ != ver: + ok = False + return False ok = True return True except Exception as e: diff --git a/modules/model_quant.py b/modules/model_quant.py index 33495b327..0bb72a75a 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -205,6 +205,9 @@ def create_sdnq_config(kwargs = None, allow: bool = True, module: str = 'Model', sdnq_config = SDNQConfig( weights_dtype=weights_dtype, group_size=shared.opts.sdnq_quantize_weights_group_size, + svd_rank=shared.opts.sdnq_svd_rank, + svd_steps=shared.opts.sdnq_svd_steps, + use_svd=shared.opts.sdnq_use_svd, quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, @@ -215,7 +218,7 @@ def create_sdnq_config(kwargs = None, allow: bool = True, module: str = 'Model', modules_to_not_convert=modules_to_not_convert, modules_dtype_dict=modules_dtype_dict.copy(), ) - log.debug(f'Quantization: module="{module}" type=sdnq mode=pre dtype={weights_dtype} matmul={shared.opts.sdnq_use_quantized_matmul} group_size={shared.opts.sdnq_quantize_weights_group_size} quant_conv={shared.opts.sdnq_quantize_conv_layers} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} dequantize_fp32={shared.opts.sdnq_dequantize_fp32} quantize_with_gpu={shared.opts.sdnq_quantize_with_gpu} quantization_device={quantization_device} return_device={return_device} device_map={shared.opts.device_map} offload_mode={shared.opts.diffusers_offload_mode} non_blocking={shared.opts.diffusers_offload_nonblocking} modules_to_not_convert={modules_to_not_convert} modules_dtype_dict={modules_dtype_dict}') + log.debug(f'Quantization: module="{module}" type=sdnq mode=pre dtype={weights_dtype} matmul={shared.opts.sdnq_use_quantized_matmul} group_size={shared.opts.sdnq_quantize_weights_group_size} svd_rank={shared.opts.sdnq_svd_rank} svd_steps={shared.opts.sdnq_svd_steps} use_svd={shared.opts.sdnq_use_svd} quant_conv={shared.opts.sdnq_quantize_conv_layers} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} dequantize_fp32={shared.opts.sdnq_dequantize_fp32} quantize_with_gpu={shared.opts.sdnq_quantize_with_gpu} quantization_device={quantization_device} return_device={return_device} device_map={shared.opts.device_map} offload_mode={shared.opts.diffusers_offload_mode} non_blocking={shared.opts.diffusers_offload_nonblocking} modules_to_not_convert={modules_to_not_convert} modules_dtype_dict={modules_dtype_dict}') if kwargs is None: return sdnq_config else: @@ -477,10 +480,10 @@ def apply_layerwise(sd_model, quiet:bool=False): def sdnq_quantize_model(model, op=None, sd_model=None, do_gc: bool = True, weights_dtype: str = None, modules_to_not_convert: list = None, modules_dtype_dict: dict = None): global quant_last_model_name, quant_last_model_device # pylint: disable=global-statement from modules import devices, shared, timer - from modules.sdnq import apply_sdnq_to_module + from modules.sdnq import sdnq_post_load_quant if weights_dtype is None: - if op is not None and ("text_encoder" in op or op in {"TE", "LLM"}) and shared.opts.sdnq_quantize_weights_mode_te not in {"Same as model", "default"}: + if (op is not None) and ("text_encoder" in op or op in {"TE", "LLM"}) and (shared.opts.sdnq_quantize_weights_mode_te not in {"Same as model", "default"}): weights_dtype = shared.opts.sdnq_quantize_weights_mode_te else: weights_dtype = shared.opts.sdnq_quantize_weights_mode @@ -495,18 +498,6 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc: bool = True, weigh if modules_dtype_dict is None: modules_dtype_dict = {} - if getattr(model, "_keep_in_fp32_modules", None) is not None: - modules_to_not_convert.extend(model._keep_in_fp32_modules) # pylint: disable=protected-access - if getattr(model, "_skip_layerwise_casting_patterns", None) is not None: - modules_to_not_convert.extend(model._skip_layerwise_casting_patterns) # pylint: disable=protected-access - if model.__class__.__name__ == "ChromaTransformer2DModel": - modules_to_not_convert.append("distilled_guidance_layer") - if model.__class__.__name__ == "QwenImageTransformer2DModel": - if "minimum_6bit" not in modules_dtype_dict.keys(): - modules_dtype_dict["minimum_6bit"] = ["img_mod", "pos_embed", "time_text_embed", "img_in", "txt_in", "norm_out"] - else: - modules_dtype_dict["minimum_6bit"].extend(["img_mod", "pos_embed", "time_text_embed", "img_in", "txt_in", "norm_out"]) - sdnq_modules_to_not_convert = [m.strip() for m in re.split(';|,| ', shared.opts.sdnq_modules_to_not_convert) if len(m.strip()) > 1] if len(sdnq_modules_to_not_convert) > 0: modules_to_not_convert.extend(sdnq_modules_to_not_convert) @@ -527,17 +518,16 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc: bool = True, weigh except Exception as e: log.warning(f'Quantization: SDNQ failed to parse sdnq_modules_dtype_dict: {e}') - model.eval() - backup_embeddings = None - if hasattr(model, "get_input_embeddings"): - backup_embeddings = copy.deepcopy(model.get_input_embeddings()) - t0 = time.time() - model = apply_sdnq_to_module( + + model = sdnq_post_load_quant( model, weights_dtype=weights_dtype, torch_dtype=devices.dtype, group_size=shared.opts.sdnq_quantize_weights_group_size, + svd_rank=shared.opts.sdnq_svd_rank, + svd_steps=shared.opts.sdnq_svd_steps, + use_svd=shared.opts.sdnq_use_svd, quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, @@ -549,12 +539,9 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc: bool = True, weigh modules_dtype_dict=modules_dtype_dict.copy(), op=op, ) + t1 = time.time() timer.load.add('sdnq', t1 - t0) - model.quantization_method = 'SDNQ' - - if hasattr(model, "set_input_embeddings") and backup_embeddings is not None: - model.set_input_embeddings(backup_embeddings) if op is not None and shared.opts.sdnq_quantize_shuffle_weights: if quant_last_model_name is not None: @@ -577,7 +564,7 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc: bool = True, weigh if do_gc: devices.torch_gc(force=True, reason='sdnq') - log.debug(f'Quantization: module="{op if op is not None else model.__class__}" type=sdnq mode=post dtype={weights_dtype} matmul={shared.opts.sdnq_use_quantized_matmul} group_size={shared.opts.sdnq_quantize_weights_group_size} quant_conv={shared.opts.sdnq_quantize_conv_layers} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} dequantize_fp32={shared.opts.sdnq_dequantize_fp32} quantize_with_gpu={shared.opts.sdnq_quantize_with_gpu} quantization_device={quantization_device} return_device={return_device} device_map={shared.opts.device_map} offload_mode={shared.opts.diffusers_offload_mode} non_blocking={shared.opts.diffusers_offload_nonblocking} modules_to_not_convert={modules_to_not_convert} modules_dtype_dict={modules_dtype_dict}') + log.debug(f'Quantization: module="{op if op is not None else model.__class__}" type=sdnq mode=post dtype={weights_dtype} matmul={shared.opts.sdnq_use_quantized_matmul} group_size={shared.opts.sdnq_quantize_weights_group_size} svd_rank={shared.opts.sdnq_svd_rank} svd_steps={shared.opts.sdnq_svd_steps} use_svd={shared.opts.sdnq_use_svd} quant_conv={shared.opts.sdnq_quantize_conv_layers} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} dequantize_fp32={shared.opts.sdnq_dequantize_fp32} quantize_with_gpu={shared.opts.sdnq_quantize_with_gpu} quantization_device={quantization_device} return_device={return_device} device_map={shared.opts.device_map} offload_mode={shared.opts.diffusers_offload_mode} non_blocking={shared.opts.diffusers_offload_nonblocking} modules_to_not_convert={modules_to_not_convert} modules_dtype_dict={modules_dtype_dict}') return model @@ -585,7 +572,7 @@ def sdnq_quantize_weights(sd_model): try: t0 = time.time() from modules import shared, devices, sd_models - log.debug(f"Quantization: type=SDNQ modules={shared.opts.sdnq_quantize_weights} dtype={shared.opts.sdnq_quantize_weights_mode} dtype_te={shared.opts.sdnq_quantize_weights_mode_te} matmul={shared.opts.sdnq_use_quantized_matmul} group_size={shared.opts.sdnq_quantize_weights_group_size} quant_conv={shared.opts.sdnq_quantize_conv_layers} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} quantize_with_gpu={shared.opts.sdnq_quantize_with_gpu} dequantize_fp32={shared.opts.sdnq_dequantize_fp32} pre_forward={shared.opts.diffusers_offload_pre}") + log.debug(f"Quantization: type=SDNQ modules={shared.opts.sdnq_quantize_weights} dtype={shared.opts.sdnq_quantize_weights_mode} dtype_te={shared.opts.sdnq_quantize_weights_mode_te} matmul={shared.opts.sdnq_use_quantized_matmul} svd_rank={shared.opts.sdnq_svd_rank} svd_steps={shared.opts.sdnq_svd_steps} use_svd={shared.opts.sdnq_use_svd} group_size={shared.opts.sdnq_quantize_weights_group_size} quant_conv={shared.opts.sdnq_quantize_conv_layers} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} quantize_with_gpu={shared.opts.sdnq_quantize_with_gpu} dequantize_fp32={shared.opts.sdnq_dequantize_fp32} pre_forward={shared.opts.diffusers_offload_pre}") global quant_last_model_name, quant_last_model_device # pylint: disable=global-statement sd_model = sd_models.apply_function_to_model(sd_model, sdnq_quantize_model, shared.opts.sdnq_quantize_weights, op="sdnq") @@ -603,6 +590,8 @@ def sdnq_quantize_weights(sd_model): log.info(f"Quantization: type=SDNQ time={t1-t0:.2f}") except Exception as e: log.warning(f"Quantization: type=SDNQ {e}") + from modules import errors + errors.display(e, 'Quantization') return sd_model @@ -610,12 +599,14 @@ def optimum_quanto_model(model, op=None, sd_model=None, weights=None, activation from modules import devices, shared quanto = load_quanto('Quantize model: type=Optimum Quanto') global quant_last_model_name, quant_last_model_device # pylint: disable=global-statement - if sd_model is not None and ("Flux" in sd_model.__class__.__name__ or "Chroma" in sd_model.__class__.__name__): # LayerNorm is not supported + if model.__class__.__name__ in {"FluxTransformer2DModel", "ChromaTransformer2DModel"}: # LayerNorm is not supported exclude_list = ["transformer_blocks.*.norm1.norm", "transformer_blocks.*.norm2", "transformer_blocks.*.norm1_context.norm", "transformer_blocks.*.norm2_context", "single_transformer_blocks.*.norm.norm", "norm_out.norm"] - if "Chroma" in sd_model.__class__.__name__: + if model.__class__.__name__ == "ChromaTransformer2DModel": # we ignore the distilled guidance layer because it degrades quality too much # see: https://github.com/huggingface/diffusers/pull/11698#issuecomment-2969717180 for more details exclude_list.append("distilled_guidance_layer.*") + elif model.__class__.__name__ == "QwenImageTransformer2DModel": + exclude_list = ["transformer_blocks.0.img_mod.1.weight", "time_text_embed", "img_in", "txt_in", "proj_out", "norm_out", "pos_embed"] else: exclude_list = None weights = getattr(quanto, weights) if weights is not None else getattr(quanto, shared.opts.optimum_quanto_weights_type) diff --git a/modules/modeldata.py b/modules/modeldata.py index c56190ad8..881a5fba0 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -62,6 +62,8 @@ def get_model_type(pipe): model_type = 'qwen' elif 'NextStep' in name: model_type = 'nextstep' + elif 'X-Omni' in name: + model_type = 'x-omni' # video models elif "CogVideo" in name: model_type = 'cogvideo' diff --git a/modules/modelloader.py b/modules/modelloader.py index 0eaccf1bd..c448caad2 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -22,10 +22,13 @@ pbar = None def hf_login(token=None): global loggedin # pylint: disable=global-statement token = token or shared.opts.huggingface_token + token = token.replace("\n", "").replace("\r", "").strip() if token is not None else None install('hf_xet', quiet=True) if token is None or len(token) <= 4: log.debug('HF login: no token provided') return False + if len(shared.opts.huggingface_mirror.strip()) > 0 and os.environ.get('HF_ENDPOINT', None) is None: + os.environ['HF_ENDPOINT'] = shared.opts.huggingface_mirror.strip() if os.environ.get('HUGGING_FACE_HUB_TOKEN', None) is not None: os.environ.pop('HUGGING_FACE_HUB_TOKEN', None) os.unsetenv('HUGGING_FACE_HUB_TOKEN') @@ -179,6 +182,13 @@ def find_diffuser(name: str, full=False): if len(repo) > 0: return [repo[0]['name']] hf_api = hf.HfApi() + suffix = '' + if len(name) > 3 and name.count('/') > 1: + parts = name.split('/') + name = '/'.join(parts[:2]) # only user/model + suffix = '/'.join(parts[2:]) # subfolder + if len(suffix) > 0: + suffix = '/' + suffix models = list(hf_api.list_models(model_name=name, library=['diffusers'], full=True, limit=20, sort="downloads", direction=-1)) if len(models) == 0: models = list(hf_api.list_models(model_name=name, full=True, limit=20, sort="downloads", direction=-1)) # widen search @@ -186,9 +196,9 @@ def find_diffuser(name: str, full=False): shared.log.debug(f'Search model: repo="{name}" {len(models) > 0}') if len(models) > 0: if not full: - return models[0].id + return models[0].id + suffix else: - return [m.id for m in models] + return [m.id + suffix for m in models] return None @@ -219,6 +229,8 @@ def get_reference_opts(name: str, quiet=False): def load_reference(name: str, variant: str = None, revision: str = None, mirror: str = None, custom_pipeline: str = None): + if '+' in name: + name = name.split('+')[0] found = [r for r in diffuser_repos if name == r['name'] or name == r['friendly'] or name == r['path']] if len(found) > 0: # already downloaded model_opts = get_reference_opts(found[0]['name']) @@ -250,7 +262,7 @@ def load_reference(name: str, variant: str = None, revision: str = None, mirror: def load_civitai(model: str, url: str): from modules import sd_models name, _ext = os.path.splitext(model) - info = sd_models.get_closet_checkpoint_match(name) + info = sd_models.get_closest_checkpoint_match(name) if info is not None: _model_opts = get_reference_opts(info.model_name) return name # already downloaded @@ -260,7 +272,7 @@ def load_civitai(model: str, url: str): download_civit_model_thread(model_name=model, model_url=url, model_path='', model_type='safetensors', token=shared.opts.civitai_token) shared.log.debug(f'Reference download complete: model="{name}"') sd_models.list_models() - info = sd_models.get_closet_checkpoint_match(name) + info = sd_models.get_closest_checkpoint_match(name) if info is not None: shared.log.debug(f'Reference: model="{name}"') return name # already downloaded @@ -294,7 +306,7 @@ def download_url_to_file(url: str, dst: str): continue break else: - shared.log.error('Error downloading: url={url} no usable temporary filename found') + shared.log.error(f'Error downloading: url={url} no usable temporary filename found') return try: with Progress(TextColumn('[cyan]{task.description}'), BarColumn(), TaskProgressColumn(), TimeRemainingColumn(), TimeElapsedColumn(), console=shared.console) as progress: diff --git a/modules/modelstats.py b/modules/modelstats.py index 2e5e7f447..2ab1a6b62 100644 --- a/modules/modelstats.py +++ b/modules/modelstats.py @@ -79,7 +79,7 @@ class Model(): return self.cls = shared.sd_model.__class__.__name__ self.type = shared.sd_model_type - self.info = sd_models.get_closet_checkpoint_match(name) + self.info = sd_models.get_closest_checkpoint_match(name) if self.info is not None: self.name = self.info.name or self.name self.hash = self.info.shorthash or '' diff --git a/modules/modular.py b/modules/modular.py new file mode 100644 index 000000000..5f3bfefc2 --- /dev/null +++ b/modules/modular.py @@ -0,0 +1,64 @@ +import time +import diffusers +from modules import shared + + +modular_map= { + 'StableDiffusionXLPipeline': 'StableDiffusionXLAutoBlocks', + 'StableDiffusionXLImg2ImgPipeline': 'StableDiffusionXLAutoBlocks', + 'StableDiffusionXLInpaintPipeline': 'StableDiffusionXLAutoBlocks', + 'FluxPipeline': 'FluxAutoBlocks', + 'FluxImg2ImgPipeline': 'FluxAutoBlocks', + 'FluxInpaintPipeline': 'FluxAutoBlocks', + 'WanPipeline': 'WanAutoBlocks', + 'WanImageToVideoPipeline': 'WanAutoBlocks', + 'QwenImagePipeline': 'QwenImageAutoBlocks', + 'QwenImageEditPipeline': 'QwenImageEditAutoBlocks', +} + + +def is_compatible(diffusion_pipeline: diffusers.DiffusionPipeline) -> bool: + if not shared.opts.model_modular_enable: + return False + compatible = diffusion_pipeline.__class__.__name__ in modular_map + if not compatible: + shared.log.debug(f'Modular: source={diffusion_pipeline.__class__.__name__} incompatible pipeline') + return compatible + + +def convert_to_modular(diffusion_pipeline: diffusers.DiffusionPipeline) -> diffusers.ModularPipeline: + modular_pipe = None + try: + t0 = time.time() + modular_cls = modular_map.get(diffusion_pipeline.__class__.__name__, None) + if modular_cls is None: + raise ValueError(f'unknown: cls={diffusion_pipeline.__class__.__name__}') + modular_cls = getattr(diffusers, modular_cls, None) + if modular_cls is None: + raise ValueError(f'invalid: cls={diffusion_pipeline.__class__.__name__}') + modular_blocks = modular_cls() + modular_pipe = modular_blocks.init_pipeline() + components_dct = {k: v for k, v in diffusion_pipeline.components.items() if v is not None} + modular_pipe.update_components(**components_dct, **diffusion_pipeline.parameters) + modular_pipe.original_pipe = diffusion_pipeline + t1 = time.time() + shared.log.debug(f'Modular: source={diffusion_pipeline.__class__.__name__} target={modular_pipe.__class__.__name__} time={t1 - t0:.2f}') + """ + for expected_input_param in modular_pipe.blocks.inputs: + name = expected_input_param.name + default = expected_input_param.default + kwargs_type = expected_input_param.kwargs_type + shared.log.trace(f'Modular input: name={name} type={kwargs_type} default={default}') + """ + + except Exception as e: + shared.log.error(f'Modular: {e}') + raise e + return modular_pipe + + +def restore_standard(modular_pipe): + if hasattr(modular_pipe, 'original_pipe'): + shared.log.debug(f'Modular: source={modular_pipe.__class__.__name__} target={modular_pipe.original_pipe.__class__.__name__}') + return modular_pipe.original_pipe + return modular_pipe diff --git a/modules/modular_guiders.py b/modules/modular_guiders.py new file mode 100644 index 000000000..556ad52b4 --- /dev/null +++ b/modules/modular_guiders.py @@ -0,0 +1,87 @@ +import diffusers +from modules import shared, errors, processing + + +# ['Default', 'CFG', 'Zero', 'PAG', 'APG', 'SLG', 'SEG', 'TCFG', 'FDG'] +guiders = { + # 'None': { 'cls': None, 'args': {}, }, + 'Default': { 'cls': None, 'args': {}, }, + 'CFG: ClassifierFreeGuidance': { 'cls': diffusers.ClassifierFreeGuidance, 'args': {} }, + 'Auto: AutoGuidance': { 'cls': diffusers.AutoGuidance, 'args': { 'dropout': 1.0, 'auto_guidance_layers': [7, 8, 9], 'auto_guidance_config': None } }, + 'Zero: ClassifierFreeZeroStar': { 'cls': diffusers.ClassifierFreeZeroStarGuidance, 'args': { 'zero_init_steps': 1 } }, + 'PAG: PerturbedAttentionGuidance': { 'cls': diffusers.PerturbedAttentionGuidance, 'args': { 'perturbed_guidance_scale': 2.8, 'perturbed_guidance_start': 0.01, 'perturbed_guidance_stop': 0.2, 'perturbed_guidance_layers': [7, 8, 9], 'perturbed_guidance_config': None } }, + 'APG: AdaptiveProjectedGuidance': { 'cls': diffusers.AdaptiveProjectedGuidance, 'args': { 'adaptive_projected_guidance_momentum': -1, 'adaptive_projected_guidance_rescale': 15.0 } }, + 'SLG: SkipLayerGuidance': { 'cls': diffusers.SkipLayerGuidance, 'args': { 'skip_layer_guidance_scale': 2.8, 'skip_layer_guidance_start': 0.01, 'skip_layer_guidance_stop': 0.2, 'skip_layer_guidance_layers': [7, 8, 9], 'skip_layer_config': None } }, + 'SEG: SmoothedEnergyGuidance': { 'cls': diffusers.SmoothedEnergyGuidance, 'args': { 'seg_guidance_scale': 3.0, 'seg_blur_sigma': 9999999.0, 'seg_blur_threshold_inf': 9999.0, 'seg_guidance_start': 0.0, 'seg_guidance_stop': 1.0, 'seg_guidance_layers': [7, 8, 9], 'seg_guidance_config': None } }, + 'TCFG: TangentialClassifierFreeGuidance': { 'cls': diffusers.TangentialClassifierFreeGuidance, 'args': {} }, + 'FDG: FrequencyDecoupledGuidance': { 'cls': diffusers.FrequencyDecoupledGuidance, 'args': { 'guidance_scales': [10.0, 5.0], 'parallel_weights': 1.0, 'guidance_rescale_space': "data" } }, +} +base_args = { + 'guidance_scale': 6.0, + 'guidance_rescale': 0.0, + 'start': 0.0, + 'stop': 1.0, +} + + +def set_guider(p: processing.StableDiffusionProcessing): + guidance_name = p.guidance_name or 'Default' + if guidance_name not in guiders: + return + + if guidance_name == 'Default': + if hasattr(shared.sd_model, 'default_guider'): + guider_info = shared.sd_model.default_guider + shared.sd_model.update_components(guider=guider_info) + else: + guider_info = shared.sd_model.get_component_spec("guider") + shared.sd_model.default_guider = guider_info + guider_cls = guider_info.type_hint + if guider_info is not None and guider_cls is not None and guider_info.config is not None: + guider_args = {k: v for k, v in guider_info.config.items() if not k.startswith('_') and v is not None} + else: + guider_args = {} + shared.log.info(f'Guider: name={guidance_name} cls={guider_cls.__name__} args={guider_args}') + return + if guidance_name == 'None': + shared.sd_model.update_components(guider=None) # breaks the pipeline + shared.log.info(f'Guider: name={guidance_name}') + return + + guider_info = guiders[guidance_name] + guider_cls = guider_info['cls'] + guider_args = {} + for k, v in base_args.items(): + if v is not None and v >= 0.0: + guider_args[k] = v + shared.log.warning('Guiders: partially implemented') # TODO: guiders + for k, v in guider_info['args'].items(): + try: + if k is None: + pass + elif k.endswith('_layers') and isinstance(v, str): + guider_args[k] = [int(x.strip()) for x in v.split(',') if x.strip().isdigit()] + elif k.endswith('_config'): + # if lsc_enabled + # guider_args[k] = diffusers.LayerSkipConfig(...) + pass + elif isinstance(v, list) and len(v) > 0: + guider_args[k] = v + elif isinstance(v, int) and (v >= 0): + guider_args[k] = int(v) + elif isinstance(v, float) and (v >= 0.0): + guider_args[k] = float(v) + elif isinstance(v, str) and (len(v) > 0): + guider_args[k] = v + except Exception as e: + shared.log.error(f'Guiders: arg={k} value={v} error={e}') + errors.display(e, 'Guiders') + # guider_args.update(guider_info['args']) + if guider_cls is not None: + try: + guider_instance = guider_cls(**guider_args) + shared.log.info(f'Guider: name={guidance_name} cls={guider_cls.__name__} args={guider_args}') + shared.sd_model.update_components(guider=guider_instance) + except Exception as e: + shared.log.error(f'Guider: name={guidance_name} cls={guider_cls.__name__} args={guider_args} {e}') + return diff --git a/modules/onnx_impl/ui.py b/modules/onnx_impl/ui.py index 0af6ba345..703392d82 100644 --- a/modules/onnx_impl/ui.py +++ b/modules/onnx_impl/ui.py @@ -15,7 +15,7 @@ def create_ui(): from modules.ui_common import create_refresh_button from modules.ui_components import DropdownMulti from modules.shared import log, opts, cmd_opts, refresh_checkpoints - from modules.sd_models import checkpoint_titles, get_closet_checkpoint_match + from modules.sd_models import checkpoint_titles, get_closest_checkpoint_match from modules.paths import sd_configs_path from .execution_providers import ExecutionProvider, install_execution_provider from .utils import check_diffusers_cache @@ -74,7 +74,7 @@ def create_ui(): cache_remove_optimized.click(fn=remove_cache_optimized, inputs=[cache_state_dirname, cache_optimized_selected,]) def cache_update_menus(query: str): - checkpoint_info = get_closet_checkpoint_match(query) + checkpoint_info = get_closest_checkpoint_match(query) if checkpoint_info is None: log.error(f"Could not find checkpoint object for '{query}'.") return diff --git a/modules/pag/__init__.py b/modules/pag/__init__.py index e67f5b086..4d92f689b 100644 --- a/modules/pag/__init__.py +++ b/modules/pag/__init__.py @@ -42,8 +42,8 @@ def apply(p: processing.StableDiffusionProcessing): # pylint: disable=arguments- pag_applied_layers_index = pag_applied_layers.split() if len(pag_applied_layers) > 0 else [] pag_applied_layers_index = [p.strip() for p in pag_applied_layers_index] p.task_args['pag_applied_layers_index'] = pag_applied_layers_index if len(pag_applied_layers_index) > 0 else ['m0'] # Available layers: d[0-5], m[0], u[0-8] - p.extra_generation_params["PAG scale"] = p.pag_scale - p.extra_generation_params["PAG adaptive"] = p.pag_adaptive + p.extra_generation_params["CFG true"] = p.pag_scale + p.extra_generation_params["CFG adaptive"] = p.pag_adaptive # shared.log.debug(f'{c}: args={p.task_args}') diff --git a/modules/postprocess/seedvr_model.py b/modules/postprocess/seedvr_model.py new file mode 100644 index 000000000..577d8609f --- /dev/null +++ b/modules/postprocess/seedvr_model.py @@ -0,0 +1,171 @@ +import time +import random +import numpy as np +import torch +from PIL import Image +from torchvision.transforms import ToPILImage +from modules import devices +from modules.shared import opts, log +from modules.upscaler import Upscaler, UpscalerData + + +MODELS_MAP = { + "SeedVR2 3B": "seedvr2_ema_3b_fp16.safetensors", + "SeedVR2 7B": "seedvr2_ema_7b_fp16.safetensors", + "SeedVR2 7B Sharp": "seedvr2_ema_7b_sharp_fp16.safetensors", +} +to_pil = ToPILImage() + + +class UpscalerSeedVR(Upscaler): + def __init__(self, dirname=None): + self.name = "SeedVR2" + super().__init__() + self.scalers = [ + UpscalerData(name="SeedVR2 3B", path=None, upscaler=self, model=None, scale=1), + UpscalerData(name="SeedVR2 7B", path=None, upscaler=self, model=None, scale=1), + UpscalerData(name="SeedVR2 7B Sharp", path=None, upscaler=self, model=None, scale=1), + ] + self.model = None + self.model_loaded = None + + def load_model(self, path: str): + model_name = MODELS_MAP.get(path, None) + if (self.model is None) or (self.model_loaded != model_name): + log.debug(f'Upscaler loading: name="{self.name}" model="{model_name}"') + t0 = time.time() + from modules.seedvr.src.core.model_manager import configure_runner + from modules.seedvr.src.core import generation + self.model = configure_runner( + model_name=model_name, + cache_dir=opts.hfcache_dir, + device=devices.device, + dtype=devices.dtype, + ) + self.model_loaded = model_name + self.model.dit.device = devices.device + self.model.dit.dtype = devices.dtype + self.model.vae_encode = self.vae_encode + self.model.vae_decode = self.vae_decode + self.model.model_step = generation.generation_step + generation.generation_step = self.model_step + self.model._internal_dict = { + 'dit': self.model.dit, + 'vae': self.model.vae, + } + t1 = time.time() + self.model.dit.config = self.model.config.dit + self.model.vae.tile_sample_min_size = 1024 + self.model.vae.tile_latent_min_size = 128 + from modules.model_quant import do_post_load_quant + self.model = do_post_load_quant(self.model, allow=True) + # from modules.sd_offload import set_diffuser_offload + # set_diffuser_offload(self.model) + log.info(f'Upscaler loaded: name="{self.name}" model="{model_name}" time={t1 - t0:.2f}') + + def vae_encode(self, samples): + log.debug(f'Upscaler encode: samples={samples[0].shape if len(samples) > 0 else None} tile={self.model.vae.tile_sample_min_size} overlap={self.model.vae.tile_overlap_factor}') + latents = [] + if len(samples) == 0: + return latents + self.model.dit = self.model.dit.to(device="cpu") + self.model.vae = self.model.vae.to(device=self.device) + devices.torch_gc() + from einops import rearrange + from modules.seedvr.src.optimization import memory_manager + memory_manager.clear_rope_cache(self.model) + scale = self.model.config.vae.scaling_factor + shift = self.model.config.vae.get("shifting_factor", 0.0) + batches = [sample.unsqueeze(0) for sample in samples] + for sample in batches: + sample = sample.to(self.device, self.model.vae.dtype) + sample = self.model.vae.preprocess(sample) + latent = self.model.vae.encode(sample).latent + latent = latent.unsqueeze(2) if latent.ndim == 4 else latent + latent = rearrange(latent, "b c ... -> b ... c") + latent = (latent - shift) * scale + latents.append(latent) + latents = [latent.squeeze(0) for latent in latents] + self.model.vae = self.model.vae.to(device="cpu") + devices.torch_gc() + return latents + + def vae_decode(self, latents, target_dtype: torch.dtype = None): + log.debug(f'Upscaler decode: latents={latents[0].shape if len(latents) > 0 else None} tile={self.model.vae.tile_latent_min_size} overlap={self.model.vae.tile_overlap_factor}') + samples = [] + if len(latents) == 0: + return samples + from einops import rearrange + from modules.seedvr.src.optimization import memory_manager + memory_manager.clear_rope_cache(self.model) + self.model.dit = self.model.dit.to(device="cpu") + self.model.vae = self.model.vae.to(device=self.device) + devices.torch_gc() + scale = self.model.config.vae.scaling_factor + shift = self.model.config.vae.get("shifting_factor", 0.0) + latents = [latent.unsqueeze(0) for latent in latents] + with devices.inference_context(): + for _i, latent in enumerate(latents): + latent = latent.to(self.device, self.model.vae.dtype) + latent = latent / scale + shift + latent = rearrange(latent, "b ... c -> b c ...") + latent = latent.squeeze(2) + sample = self.model.vae.decode(latent).sample + sample = self.model.vae.postprocess(sample) + samples.append(sample) + samples = [sample.squeeze(0) for sample in samples] + self.model.vae = self.model.vae.to(device="cpu") + devices.torch_gc() + return samples + + def model_step(self, *args, **kwargs): + from modules.seedvr.src.optimization import memory_manager + self.model.vae = self.model.vae.to(device="cpu") + self.model.dit = self.model.dit.to(device=self.device) + devices.torch_gc() + log.debug(f'Upscaler inference: args={len(args)} kwargs={list(kwargs.keys())}') + memory_manager.preinitialize_rope_cache(self.model) + with devices.inference_context(): + result = self.model.model_step(*args, **kwargs) + self.model.dit = self.model.dit.to(device="cpu") + devices.torch_gc() + return result + + def do_upscale(self, img: Image.Image, selected_file): + self.load_model(selected_file) + if self.model is None: + return img + + from modules.seedvr.src.core import generation + + width = int(self.scale * img.width) // 8 * 8 + image_tensor = np.array(img) + image_tensor = torch.from_numpy(image_tensor).to(device=devices.device, dtype=devices.dtype).unsqueeze(0) / 255.0 + + random.seed() + seed = int(random.randrange(4294967294)) + + t0 = time.time() + with devices.inference_context(): + result_tensor = generation.generation_loop( + runner=self.model, + images=image_tensor, + cfg_scale=opts.seedvt_cfg_scale, + seed=seed, + res_w=width, + batch_size=1, + temporal_overlap=0, + device=devices.device, + ) + t1 = time.time() + log.info(f'Upscaler: type="{self.name}" model="{selected_file}" scale={self.scale} cfg={opts.seedvt_cfg_scale} seed={seed} time={t1 - t0:.2f}') + img = to_pil(result_tensor.squeeze().permute((2, 0, 1))) + + if opts.upscaler_unload: + self.model.dit = None + self.model.vae = None + self.model.cache = None + self.model = None + log.debug(f'Upscaler unload: type="{self.name}" model="{selected_file}"') + devices.torch_gc(force=True) + return img diff --git a/modules/processing.py b/modules/processing.py index 31378fa4a..aaf681840 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -46,7 +46,7 @@ class Processed: self.sampler_name = p.sampler_name or '' self.cfg_scale = p.cfg_scale if p.cfg_scale > 1 else None - self.cfg_end = p.cfg_end if p.cfg_end < 0 else None + self.cfg_end = p.cfg_end if p.cfg_end < 1 else None self.image_cfg_scale = p.image_cfg_scale or 0 self.steps = p.steps or 0 self.batch_size = max(1, p.batch_size) @@ -130,7 +130,7 @@ def get_processed(*args, **kwargs): def process_images(p: StableDiffusionProcessing) -> Processed: timer.process.reset() - debug(f'Process images: {vars(p)}') + debug(f'Process images: class={p.__class__.__name__} {vars(p)}') if not hasattr(p.sd_model, 'sd_checkpoint_info'): shared.log.error('Processing: incomplete model') return None @@ -427,13 +427,20 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: for script_image, script_infotext in zip(results.images, results.infotexts): output_images.append(script_image) infotexts.append(script_infotext) + if samples is None: from modules.processing_diffusers import process_diffusers samples = process_diffusers(p) timer.process.record('process') - if not shared.opts.keep_incomplete and shared.state.interrupted: - samples = [] + if shared.state.interrupted: + shared.log.debug(f'Process: batch={n+1}/{p.n_iter} interrupted') + p.do_not_save_samples = not shared.opts.keep_incomplete + if shared.state.current_image is not None and isinstance(shared.state.current_image, Image.Image): + samples = [shared.state.current_image] + infotexts = [create_infotext(p, p.all_prompts, p.all_seeds, p.all_subseeds, index=0)] + else: + samples = [] if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner): p.scripts.postprocess_batch(p, samples, batch_number=n) diff --git a/modules/processing_args.py b/modules/processing_args.py index 5d40d8db0..3e28e158a 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -18,6 +18,26 @@ debug_log = shared.log.trace if debug_enabled else lambda *args, **kwargs: None disable_pbar = os.environ.get('SD_DISABLE_PBAR', None) is not None +def task_modular_kwargs(p, model): # pylint: disable=unused-argument + # model_cls = model.__class__.__name__ + task_args = {} + p.ops.append('modular') + + processing_helpers.resize_init_images(p) + task_args['width'] = p.width + task_args['height'] = p.height + if len(getattr(p, 'init_images', [])) > 0: + task_args['image'] = p.init_images + task_args['strength'] = p.denoising_strength + mask_image = p.task_args.get('image_mask', None) or getattr(p, 'image_mask', None) or getattr(p, 'mask', None) + if mask_image is not None: + task_args['mask_image'] = mask_image + + if debug_enabled: + debug_log(f'Process task specific args: {task_args}') + return task_args + + def task_specific_kwargs(p, model): model_cls = model.__class__.__name__ vae_scale_factor = sd_vae.get_vae_scale_factor(model) @@ -103,9 +123,11 @@ def task_specific_kwargs(p, model): } # model specific args - if model_cls == 'QwenImageEditPipeline' and len(getattr(p, 'init_images', [])) == 0: + if 'QwenImageEdit' in model_cls and (p.init_images is None or len(p.init_images) == 0): task_args['image'] = [Image.new('RGB', (p.width, p.height), (0, 0, 0))] # monkey-patch so qwen-image-edit pipeline does not error-out on t2i - if model_cls == 'LatentConsistencyModelPipeline' and hasattr(p, 'init_images') and len(p.init_images) > 0: + if 'QwenImageEditPlusPipeline' in model_cls and p.init_control is not None and len(p.init_control) > 0: + task_args['image'] += p.init_control + if 'LatentConsistencyModelPipeline' in model_cls and len(p.init_images) > 0: p.ops.append('lcm') init_latents = [processing_vae.vae_encode(image, model=shared.sd_model, vae_type=p.vae_type).squeeze(dim=0) for image in p.init_images] init_latent = torch.stack(init_latents, dim=0).to(shared.device) @@ -113,11 +135,11 @@ def task_specific_kwargs(p, model): init_latent = (1 - p.denoising_strength) * init_latent + init_noise task_args = { 'latents': init_latent.to(model.dtype), - 'width': p.width if hasattr(p, 'width') else None, - 'height': p.height if hasattr(p, 'height') else None, + 'width': p.width, + 'height': p.height, } - if model_cls == 'BlipDiffusionPipeline': - if len(getattr(p, 'init_images', [])) == 0: + if 'BlipDiffusionPipeline' in model_cls: + if len(p.init_images) == 0: shared.log.error('BLiP diffusion requires init image') return task_args task_args = { @@ -126,14 +148,26 @@ def task_specific_kwargs(p, model): 'target_subject_category': getattr(p, 'prompt', '').split()[-1], 'output_type': 'pil', } - if model.__class__.__name__ == 'WanImageToVideoPipeline' and hasattr(p, 'init_images') and len(p.init_images) > 0: + if ('WanImageToVideoPipeline' in model_cls) and (p.init_images is not None) and (len(p.init_images) > 0): task_args['image'] = p.init_images[0] + if ('WanVACEPipeline' in model_cls) and (p.init_images is not None) and (len(p.init_images) > 0): + task_args['reference_images'] = p.init_images if debug_enabled: debug_log(f'Process task specific args: {task_args}') return task_args +def get_params(model): + if hasattr(model, 'blocks') and hasattr(model.blocks, 'inputs'): # modular pipeline + possible = [input_param.name for input_param in model.blocks.inputs] + return possible + else: + signature = inspect.signature(type(model).__call__, follow_wrapped=True) + possible = list(signature.parameters) + return possible + + def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:typing.Optional[list]=None, negative_prompts_2:typing.Optional[list]=None, prompt_attention:typing.Optional[str]=None, desc:typing.Optional[str]='', **kwargs): t0 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) @@ -149,8 +183,8 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba', disable=disable_pbar) else: model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba') - signature = inspect.signature(type(model).__call__, follow_wrapped=True) - possible = list(signature.parameters) + + possible = get_params(model) if debug_enabled: debug_log(f'Process pipeline possible: {possible}') @@ -307,6 +341,14 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t args['control_strength'] = p.denoising_strength args['width'] = p.width args['height'] = p.height + if 'WanVACEPipeline' in model.__class__.__name__: + if isinstance(args['prompt'], list): + args['prompt'] = args['prompt'][0] if len(args['prompt']) > 0 else '' + if isinstance(args.get('negative_prompt', None), list): + args['negative_prompt'] = args['negative_prompt'][0] if len(args['negative_prompt']) > 0 else '' + if isinstance(args['generator'], list) and len(args['generator']) > 0: + args['generator'] = args['generator'][0] + # set callbacks if 'prior_callback_steps' in possible: # Wuerstchen / Cascade args['prior_callback_steps'] = 1 @@ -347,7 +389,11 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t args[arg] = kwargs[arg] # handle task specific args - task_kwargs = task_specific_kwargs(p, model) + if sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.MODULAR: + task_kwargs = task_modular_kwargs(p, model) + else: + task_kwargs = task_specific_kwargs(p, model) + pipe_args = getattr(p, 'task_args', {}) model_args = getattr(model, 'task_args', {}) task_kwargs.update(pipe_args or {}) @@ -397,8 +443,9 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t # handle implicit controlnet if 'control_image' in possible and 'control_image' not in args and 'image' in args: - debug_log('Process: set control image') - args['control_image'] = args['image'] + if sd_models.get_diffusers_task(model) != sd_models.DiffusersTaskType.MODULAR: + debug_log('Process: set control image') + args['control_image'] = args['image'] sd_hijack_hypertile.hypertile_set(p, hr=len(getattr(p, 'init_images', [])) > 0) diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 46067e0c2..62f9b624b 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -1,6 +1,5 @@ import typing import os -import sys import time import torch import numpy as np @@ -56,7 +55,7 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} t0 = time.time() if devices.backend == "ipex": # xe driver on linux needs this torch.xpu.synchronize(devices.device) - elif devices.backend == "zluda" or (devices.backend == "rocm" and sys.platform == "win32"): + elif (devices.backend == "zluda") or (devices.backend == "rocm") or (devices.backend == "cuda"): torch.cuda.synchronize(devices.device) latents = kwargs.get('latents', None) if debug: diff --git a/modules/processing_class.py b/modules/processing_class.py index 7ea9c2eab..8ac7286fc 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -36,7 +36,13 @@ class StableDiffusionProcessing: sampler_name: str = None, hr_sampler_name: str = None, eta: float = None, - # guidance + # modular guidance + guidance_name: str = 'Default', + guidance_scale: float = 6.0, + guidance_rescale: float = 0.0, + guidance_start: float = 0.0, + guidance_stop: float = 1.0, + # legacy guidance cfg_scale: float = 6.0, cfg_end: float = 1, diffusers_guidance_rescale: float = 0.0, @@ -72,7 +78,8 @@ class StableDiffusionProcessing: hdr_color_picker: str = None, hdr_tint_ratio: float = 0, # img2img - init_images: list = None, + init_images: list = [], + init_control: list = [], denoising_strength: float = 0.3, image_cfg_scale: float = None, initial_noise_multiplier: float = None, # pylint: disable=unused-argument # a1111 compatibility @@ -208,6 +215,7 @@ class StableDiffusionProcessing: self.detailer_resolution = detailer_resolution self.restore_faces = restore_faces self.init_images = init_images + self.init_control = init_control self.resize_mode = resize_mode self.resize_name = resize_name self.resize_context = resize_context @@ -247,6 +255,11 @@ class StableDiffusionProcessing: self.do_not_save_grid = do_not_save_grid self.override_settings_restore_afterwards = override_settings_restore_afterwards self.eta = eta + self.guidance_name = guidance_name + self.guidance_scale = guidance_scale + self.guidance_rescale = guidance_rescale + self.guidance_start = guidance_start + self.guidance_stop = guidance_stop self.cfg_scale = cfg_scale self.cfg_end = cfg_end self.diffusers_guidance_rescale = diffusers_guidance_rescale @@ -445,14 +458,14 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.hr_upscale_to_y = int(self.height * scale) else: if self.hr_resize_y == 0: - self.hr_upscale_to_x = self.hr_resize_x - self.hr_upscale_to_y = self.hr_resize_x * self.height // self.width + self.hr_upscale_to_x = int(self.hr_resize_x) + self.hr_upscale_to_y = int(self.hr_resize_x * self.height // self.width) elif self.hr_resize_x == 0: - self.hr_upscale_to_x = self.hr_resize_y * self.width // self.height - self.hr_upscale_to_y = self.hr_resize_y + self.hr_upscale_to_x = int(self.hr_resize_y * self.width // self.height) + self.hr_upscale_to_y = int(self.hr_resize_y) elif self.hr_resize_x > 0 and self.hr_resize_y > 0: - self.hr_upscale_to_x = self.hr_resize_x - self.hr_upscale_to_y = self.hr_resize_y + self.hr_upscale_to_x = int(self.hr_resize_x) + self.hr_upscale_to_y = int(self.hr_resize_y) shared.log.debug(f'Init hires: upscaler="{self.hr_upscaler}" sampler="{self.hr_sampler_name}" resize={self.hr_resize_x}x{self.hr_resize_y} upscale={self.hr_upscale_to_x}x{self.hr_upscale_to_y}') @@ -572,9 +585,9 @@ class StableDiffusionProcessingControl(StableDiffusionProcessingImg2Img): self.hr_force = force if use_scale: vae_scale_factor = sd_vae.get_vae_scale_factor() - self.hr_upscale_to_x, self.hr_upscale_to_y = vae_scale_factor * int(self.width * scale / vae_scale_factor), vae_scale_factor * int(self.height * scale / vae_scale_factor) + self.hr_upscale_to_x, self.hr_upscale_to_y = int(vae_scale_factor * int(self.width * scale / vae_scale_factor)), int(vae_scale_factor * int(self.height * scale / vae_scale_factor)) else: - self.hr_upscale_to_x, self.hr_upscale_to_y = self.hr_resize_x, self.hr_resize_y + self.hr_upscale_to_x, self.hr_upscale_to_y = int(self.hr_resize_x), int(self.hr_resize_y) def switch_class(p: StableDiffusionProcessing, new_class: type, dct: dict = None): diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 8353be172..277b5e1c5 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -67,7 +67,7 @@ def restore_state(p: processing.StableDiffusionProcessing): def process_pre(p: processing.StableDiffusionProcessing): - from modules import ipadapter, sd_hijack_freeu, para_attention, teacache, hidiffusion, ras, pag, cfgzero, transformer_cache, token_merge, linfusion + from modules import ipadapter, sd_hijack_freeu, para_attention, teacache, hidiffusion, ras, pag, cfgzero, transformer_cache, token_merge, linfusion, cachedit shared.log.info('Processing modifiers: apply') try: @@ -80,6 +80,7 @@ def process_pre(p: processing.StableDiffusionProcessing): pag.apply(p) cfgzero.apply(p) linfusion.apply(shared.sd_model) + cachedit.apply_cache_dit(shared.sd_model) # apply-only sd_hijack_freeu.apply_freeu(p) @@ -95,11 +96,20 @@ def process_pre(p: processing.StableDiffusionProcessing): # sd_models.move_model(shared.sd_model.unet, devices.device) # if hasattr(shared.sd_model, 'transformer'): # sd_models.move_model(shared.sd_model.transformer, devices.device) + + from modules import modular + if modular.is_compatible(shared.sd_model): + modular_pipe = modular.convert_to_modular(shared.sd_model) + if modular_pipe is not None: + shared.sd_model = modular_pipe + from modules import modular_guiders + modular_guiders.set_guider(p) + timer.process.record('pre') def process_post(p: processing.StableDiffusionProcessing): - from modules import ipadapter, hidiffusion, ras, pag, cfgzero, token_merge, linfusion + from modules import ipadapter, hidiffusion, ras, pag, cfgzero, token_merge, linfusion, cachedit shared.log.info('Processing modifiers: unapply') try: @@ -111,6 +121,7 @@ def process_post(p: processing.StableDiffusionProcessing): pag.unapply() cfgzero.unapply() linfusion.unapply(shared.sd_model) + cachedit.unapply_cache_dir(shared.sd_model) except Exception as e: shared.log.error(f'Processing unapply: {e}') errors.display(e, 'unapply') @@ -229,8 +240,8 @@ def process_hires(p: processing.StableDiffusionProcessing, output): p.hr_upscaler = p.resize_name p.hr_resize_mode = p.resize_mode p.hr_resize_context = p.resize_context - p.hr_upscale_to_x = p.width * p.hr_scale if p.hr_resize_x == 0 else p.hr_resize_x - p.hr_upscale_to_y = p.height * p.hr_scale if p.hr_resize_y == 0 else p.hr_resize_y + p.hr_upscale_to_x = int(p.width * p.hr_scale) if p.hr_resize_x == 0 else p.hr_resize_x + p.hr_upscale_to_y = int(p.height * p.hr_scale) if p.hr_resize_y == 0 else p.hr_resize_y # hires runs on original pipeline if hasattr(shared.sd_model, 'restore_pipeline') and (shared.sd_model.restore_pipeline is not None) and (not shared.opts.control_hires): @@ -487,7 +498,7 @@ def validate_pipeline(p: processing.StableDiffusionProcessing): if m.repo_cls is not None: models_cls.append(m.repo_cls.__name__) is_video_model = shared.sd_model.__class__.__name__ in models_cls - override_video_pipelines = ['WanPipeline', 'WanImageToVideoPipeline'] + override_video_pipelines = ['WanPipeline', 'WanImageToVideoPipeline', 'WanVACEPipeline'] is_video_pipeline = ('video' in p.__class__.__name__.lower()) or (shared.sd_model.__class__.__name__ in override_video_pipelines) if is_video_model and not is_video_pipeline: shared.log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} video model with non-video pipeline') diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index 976ab0c32..b58f0594a 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -9,6 +9,7 @@ import cv2 from PIL import Image from blendmodes.blend import blendLayers, BlendType from modules import shared, devices, images, sd_models, sd_samplers, sd_vae, sd_hijack_hypertile, processing_vae, timer +from modules.api import helpers debug = shared.log.trace if os.environ.get('SD_PROCESS_DEBUG', None) is not None else lambda *args, **kwargs: None @@ -16,6 +17,10 @@ debug_steps = shared.log.trace if os.environ.get('SD_STEPS_DEBUG', None) is not debug_steps('Trace: STEPS') +def is_modular(): + return sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.MODULAR + + def is_txt2img(): return sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE @@ -275,10 +280,38 @@ def validate_sample(tensor): return cast +def decode_images(image): + if isinstance(image, list): + decoded = [] + for i, img in enumerate(image): + if isinstance(img, str): + try: + decoded.append(helpers.decode_base64_to_image(img, quiet=True)) + except Exception as e: + shared.log.error(f'Decode image[{i}]: {e}') + elif isinstance(img, Image.Image): + decoded.append(img) + else: + shared.log.error(f'Decode image[{i}]: {type(img)} unknown type') + return decoded + elif isinstance(image, str): + try: + return helpers.decode_base64_to_image(image, quiet=True) + except Exception as e: + shared.log.error(f'Decode image: {e}') + elif isinstance(image, Image.Image): + return image + else: + shared.log.error(f'Decode image: {type(image)} unknown type') + return None + + def resize_init_images(p): if getattr(p, 'image', None) is not None and getattr(p, 'init_images', None) is None: p.init_images = [p.image] + if getattr(p, 'init_images', None) is not None and len(p.init_images) > 0: + p.init_images = decode_images(p.init_images) vae_scale_factor = sd_vae.get_vae_scale_factor() tgt_width, tgt_height = vae_scale_factor * math.ceil(p.init_images[0].width / vae_scale_factor), vae_scale_factor * math.ceil(p.init_images[0].height / vae_scale_factor) if p.init_images[0].size != (tgt_width, tgt_height): @@ -287,11 +320,14 @@ def resize_init_images(p): p.height = tgt_height p.width = tgt_width sd_hijack_hypertile.hypertile_set(p) - if getattr(p, 'mask', None) is not None and p.mask.size != (tgt_width, tgt_height): + if getattr(p, 'mask', None) is not None and p.mask is not None and p.mask.size != (tgt_width, tgt_height): + p.mask = decode_images(p.mask) p.mask = images.resize_image(1, p.mask, tgt_width, tgt_height, upscaler_name=None) - if getattr(p, 'init_mask', None) is not None and p.init_mask.size != (tgt_width, tgt_height): + if getattr(p, 'init_mask', None) is not None and p.init_mask is not None and p.init_mask.size != (tgt_width, tgt_height): + p.init_mask = decode_images(p.init_mask) p.init_mask = images.resize_image(1, p.init_mask, tgt_width, tgt_height, upscaler_name=None) - if getattr(p, 'mask_for_overlay', None) is not None and p.mask_for_overlay.size != (tgt_width, tgt_height): + if getattr(p, 'mask_for_overlay', None) is not None and p.mask_for_overlay is not None and p.mask_for_overlay.size != (tgt_width, tgt_height): + p.mask_for_overlay = decode_images(p.mask_for_overlay) p.mask_for_overlay = images.resize_image(1, p.mask_for_overlay, tgt_width, tgt_height, upscaler_name=None) return tgt_width, tgt_height return p.width, p.height @@ -372,7 +408,9 @@ def calculate_base_steps(p, use_denoise_start, use_refiner_start): if len(getattr(p, 'timesteps', [])) > 0: return None cls = shared.sd_model.__class__.__name__ - if 'Flex' in cls or 'Kontext' in cls or 'Edit' in cls: + if 'Flex' in cls or 'Kontext' in cls or 'Edit' in cls or 'Wan' in cls: + steps = p.steps + elif is_modular(): steps = p.steps elif not is_txt2img(): if cls in sd_models.i2i_pipes: @@ -393,7 +431,7 @@ def calculate_base_steps(p, use_denoise_start, use_refiner_start): def calculate_hires_steps(p): cls = shared.sd_model.__class__.__name__ - if 'Flex' in cls or 'HiDreamImageEditingPipeline' in cls or 'Kontext' in cls: + if 'Flex' in cls or 'Kontext' in cls or 'Edit' in cls or 'Wan' in cls: steps = p.steps elif p.hr_second_pass_steps > 0: steps = (p.hr_second_pass_steps // p.denoising_strength) + 1 @@ -407,7 +445,7 @@ def calculate_hires_steps(p): def calculate_refiner_steps(p): cls = shared.sd_model.__class__.__name__ - if 'Flex' in cls or 'HiDreamImageEditingPipeline' in cls or 'Kontext' in cls: + if 'Flex' in cls or 'Kontext' in cls or 'Edit' in cls or 'Wan' in cls: steps = p.steps elif "StableDiffusionXL" in shared.sd_refiner.__class__.__name__: if p.refiner_start > 0 and p.refiner_start < 1: diff --git a/modules/processing_info.py b/modules/processing_info.py index 694fd52b3..cb1050349 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -31,13 +31,13 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No if all_subseeds is None: all_subseeds = p.all_subseeds or [p.subseed] while len(all_prompts) <= index: - all_prompts.append(all_prompts[-1]) + all_prompts.insert(0, p.prompt) while len(all_seeds) <= index: - all_seeds.append(all_seeds[-1]) + all_seeds.insert(0, int(p.seed)) while len(all_subseeds) <= index: - all_subseeds.append(all_subseeds[-1]) + all_subseeds.insert(0, int(p.subseed)) while len(all_negative_prompts) <= index: - all_negative_prompts.append(all_negative_prompts[-1]) + all_negative_prompts.insert(0, p.negative_prompt) comment = ', '.join(comments) if comments is not None and type(comments) is list else None ops = list(set(p.ops)) args = { @@ -50,7 +50,8 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No "CFG scale": p.cfg_scale if p.cfg_scale > 1.0 else None, "CFG rescale": p.diffusers_guidance_rescale if p.diffusers_guidance_rescale > 0 else None, "CFG end": p.cfg_end if p.cfg_end < 1.0 else None, - "CFG true": p.pag_scale if p.pag_scale > 1 else None, + "CFG true": p.pag_scale if p.pag_scale > 0 else None, + "CFG adaptive": p.pag_adaptive if p.pag_adaptive != 0.5 else None, "Clip skip": p.clip_skip if p.clip_skip > 1 else None, "Batch": f'{p.n_iter}x{p.batch_size}' if p.n_iter > 1 or p.batch_size > 1 else None, "Refiner prompt": p.refiner_prompt if len(p.refiner_prompt) > 0 else None, diff --git a/modules/processing_vae.py b/modules/processing_vae.py index aca549a39..00ffbb450 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -117,8 +117,10 @@ def full_vae_decode(latents, model): scaling_factor = model.vae.config.get("scaling_factor", 1.0) shift_factor = model.vae.config.get("shift_factor", None) if latents_mean and latents_std: - latents_mean = (torch.tensor(latents_mean).view(1, -1, 1, 1).to(latents.device, latents.dtype)) - latents_std = (torch.tensor(latents_std).view(1, -1, 1, 1).to(latents.device, latents.dtype)) + broadcast_shape = [1 for _ in range(latents.ndim)] + broadcast_shape[1] = -1 + latents_mean = (torch.tensor(latents_mean).view(*broadcast_shape).to(latents.device, latents.dtype)) + latents_std = (torch.tensor(latents_std).view(*broadcast_shape).to(latents.device, latents.dtype)) latents = ((latents * latents_std) / scaling_factor) + latents_mean else: latents = latents / scaling_factor @@ -173,7 +175,7 @@ def full_vae_decode(latents, model): log_debug(f'VAE memory: {shared.mem_mon.read()}') vae_name = os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0] if sd_vae.loaded_vae_file is not None else "default" vae_scale_factor = sd_vae.get_vae_scale_factor(model) - shared.log.debug(f'Decode: vae="{vae_name}" scale={vae_scale_factor} upcast={upcast} slicing={getattr(model.vae, "use_slicing", None)} tiling={getattr(model.vae, "use_tiling", None)} latents={list(latents.shape)}:{latents.device}:{latents.dtype} time={t1-t0:.3f}') + shared.log.debug(f'Decode: vae="{vae_name}" scale={vae_scale_factor} upcast={upcast} slicing={getattr(model.vae, "use_slicing", None)} tiling={getattr(model.vae, "use_tiling", None)} latents={list(latents.shape)}:{latents.device} dtype={latents.dtype} time={t1-t0:.3f}') return decoded @@ -221,7 +223,7 @@ def taesd_vae_decode(latents): else: decoded = sd_vae_taesd.decode(latents) t1 = time.time() - shared.log.debug(f'Decode: vae="taesd" latents={latents.shape}:{latents.dtype}:{latents.device} time={t1-t0:.3f}') + shared.log.debug(f'Decode: vae="taesd" latents={latents.shape}:{latents.device} dtype={latents.dtype} time={t1-t0:.3f}') return decoded @@ -237,13 +239,15 @@ def vae_postprocess(tensor, model, output_type='np'): if isinstance(tensor, list) and len(tensor) > 0 and torch.is_tensor(tensor[0]): tensor = torch.stack(tensor) if torch.is_tensor(tensor): - if len(tensor.shape) == 3 and tensor.shape[0] == 3: + if tensor.ndim == 3 and tensor.shape[0] == 3: tensor = tensor.unsqueeze(0) if hasattr(model, 'video_processor'): - if len(tensor.shape) == 6 and tensor.shape[1] == 1: + if tensor.ndim == 6 and tensor.shape[1] == 1: tensor = tensor.squeeze(0) images = model.video_processor.postprocess_video(tensor, output_type='pil') elif hasattr(model, 'image_processor'): + if tensor.ndim == 5 and tensor.shape[1] == 3: # Qwen Image + tensor = tensor[:, :, 0] images = model.image_processor.postprocess(tensor, output_type=output_type) elif hasattr(model, "vqgan"): images = tensor.permute(0, 2, 3, 1).cpu().float().numpy() @@ -252,6 +256,8 @@ def vae_postprocess(tensor, model, output_type='np'): else: from diffusers.image_processor import VaeImageProcessor model.image_processor = VaeImageProcessor() + if tensor.ndim == 5 and tensor.shape[1] == 3: # Qwen Image + tensor = tensor[:, :, 0] images = model.image_processor.postprocess(tensor, output_type=output_type) else: images = tensor if isinstance(tensor, list) or isinstance(tensor, np.ndarray) else [tensor] @@ -291,8 +297,9 @@ def vae_decode(latents, model, output_type='np', vae_type='Full', width=None, he latent_num_frames = (frames - 1) // model.vae_temporal_compression_ratio + 1 latents = model._unpack_latents(latents.unsqueeze(0), latent_num_frames, height // 32, width // 32, model.transformer_spatial_patch_size, model.transformer_temporal_patch_size) # pylint: disable=protected-access latents = model._denormalize_latents(latents, model.vae.latents_mean, model.vae.latents_std, model.vae.config.scaling_factor) # pylint: disable=protected-access - if hasattr(model, '_unpack_latents') and hasattr(model, "vae_scale_factor") and width is not None and height is not None and latents.ndim == 3: # FLUX + elif hasattr(model, '_unpack_latents') and hasattr(model, "vae_scale_factor") and width is not None and height is not None and latents.ndim == 3: # FLUX latents = model._unpack_latents(latents, height, width, model.vae_scale_factor) # pylint: disable=protected-access + if latents.ndim == 3: # lost a batch dim in hires latents = latents.unsqueeze(0) if latents.shape[-1] <= 4: # not a latent, likely an image diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 31947ee7b..9ac43de40 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -59,12 +59,12 @@ class PromptEmbedder: self.steps = steps self.clip_skip = clip_skip # All embeds are nested lists, outer list batch length, inner schedule length - self.prompt_embeds = [[]] * self.batchsize - self.positive_pooleds = [[]] * self.batchsize - self.negative_prompt_embeds = [[]] * self.batchsize - self.negative_pooleds = [[]] * self.batchsize - self.prompt_attention_masks = [[]] * self.batchsize - self.negative_prompt_attention_masks = [[]] * self.batchsize + self.prompt_embeds = [[] for _ in range(self.batchsize)] + self.positive_pooleds = [[] for _ in range(self.batchsize)] + self.negative_prompt_embeds = [[] for _ in range(self.batchsize)] + self.negative_pooleds = [[] for _ in range(self.batchsize)] + self.prompt_attention_masks = [[] for _ in range(self.batchsize)] + self.negative_prompt_attention_masks = [[] for _ in range(self.batchsize)] self.positive_schedule = None self.negative_schedule = None self.scheduled_prompt = False @@ -77,13 +77,25 @@ class PromptEmbedder: if self.pipe is None: shared.log.error("Prompt encode: cannot find text encoder in model") return + seen_prompts = {} # per prompt in batch for batchidx, (prompt, negative_prompt) in enumerate(zip(self.prompts, self.negative_prompts)): self.prepare_schedule(prompt, negative_prompt) + schedule_key = ( + tuple(self.positive_schedule) if self.positive_schedule is not None else None, + tuple(self.negative_schedule) if self.negative_schedule is not None else None, + self.scheduled_prompt, + ) + cache_key = (prompt, negative_prompt, schedule_key) + cached_idx = seen_prompts.get(cache_key) + if cached_idx is not None: + self.clone_embeds(batchidx, cached_idx) + continue if self.scheduled_prompt: self.scheduled_encode(self.pipe, batchidx) else: self.encode(self.pipe, prompt, negative_prompt, batchidx) + seen_prompts[cache_key] = batchidx self.checkcache(p) debug(f"Prompt encode: time={(time.time() - t0):.3f}") @@ -207,22 +219,42 @@ class PromptEmbedder: negative_pooled, negative_prompt_attention_mask ) = get_weighted_text_embeddings(pipe, positive_prompt, negative_prompt, self.clip_skip) - if prompt_embed is not None: - self.prompt_embeds[batchidx] = [prompt_embed] - if negative_embed is not None: - self.negative_prompt_embeds[batchidx] = [negative_embed] - if positive_pooled is not None: - self.positive_pooleds[batchidx] = [positive_pooled] - if negative_pooled is not None: - self.negative_pooleds[batchidx] = [negative_pooled] - if prompt_attention_mask is not None: - self.prompt_attention_masks[batchidx] = [prompt_attention_mask] - if negative_prompt_attention_mask is not None: - self.negative_prompt_attention_masks[batchidx] = [negative_prompt_attention_mask] + def _store(target, value): + if value is None: + return + # scheduled prompts need to keep all slices, unscheduled can overwrite + if self.scheduled_prompt and len(target[batchidx]) > 0: + target[batchidx].append(value) + else: + target[batchidx] = [value] + + _store(self.prompt_embeds, prompt_embed) + _store(self.negative_prompt_embeds, negative_embed) + _store(self.positive_pooleds, positive_pooled) + _store(self.negative_pooleds, negative_pooled) + _store(self.prompt_attention_masks, prompt_attention_mask) + _store(self.negative_prompt_attention_masks, negative_prompt_attention_mask) if debug_enabled: get_tokens(pipe, 'positive', positive_prompt) get_tokens(pipe, 'negative', negative_prompt) + def clone_embeds(self, batchidx, idx): + def _clone(target): + if len(target) <= idx: + return + src = target[idx] + if isinstance(src, list): + target[batchidx] = [item if not isinstance(item, list) else list(item) for item in src] + else: + target[batchidx] = src + + _clone(self.prompt_embeds) + _clone(self.negative_prompt_embeds) + _clone(self.positive_pooleds) + _clone(self.negative_pooleds) + _clone(self.prompt_attention_masks) + _clone(self.negative_prompt_attention_masks) + def __call__(self, key, step=0): batch = getattr(self, key) res = [] @@ -589,6 +621,9 @@ def get_weighted_text_embeddings(pipe, prompt: str = "", neg_prompt: str = "", c negative_weights.pop(0) embedding_providers = prepare_embedding_providers(pipe, clip_skip) + if len(embedding_providers) == 0: + shared.log.error("Prompt encode: cannot find text encoder in model") + return None, None, None, None, None, None empty_embedding_providers = None if 'StableCascade' in pipe.__class__.__name__: empty_embedding_providers = [embedding_providers[1]] diff --git a/modules/rocm.py b/modules/rocm.py index 554c3af55..1b03bfd32 100644 --- a/modules/rocm.py +++ b/modules/rocm.py @@ -3,9 +3,9 @@ import sys import ctypes import shutil import subprocess -import importlib.metadata from typing import Union, List from enum import Enum +from functools import wraps def resolve_link(path_: str) -> str: @@ -29,15 +29,30 @@ def load_library_global(path_: str): ctypes.CDLL(path_, mode=ctypes.RTLD_GLOBAL) -def conceal(): - os.environ.pop("ROCM_HOME", None) - os.environ.pop("ROCM_PATH", None) - paths = os.environ["PATH"].split(";") - paths_no_rocm = [] - for path_ in paths: - if "rocm" not in path_.lower(): - paths_no_rocm.append(path_) - os.environ["PATH"] = ";".join(paths_no_rocm) +class Environment: + pass + + +# rocm is installed system-wide +class ROCmEnvironment(Environment): + path: str + + def __init__(self, path: str): + self.path = path + + +# rocm-sdk package is installed +class PythonPackageEnvironment(Environment): + hip: ctypes.CDLL + + def __init__(self): + import _rocm_sdk_core + if sys.platform == "win32": + path = os.path.join(_rocm_sdk_core.__path__[0], "bin", "amdhip64_7.dll") + else: + raise NotImplementedError + # This library will be loaded/used by PyTorch. So it won't make conflicts. + self.hip = ctypes.CDLL(path) class MicroArchitecture(Enum): @@ -80,8 +95,22 @@ class Agent: self.is_apu = (self.gfx_version & 0xFFF0 == 0x1150) or self.gfx_version in (0x801, 0x902, 0x90c, 0x1013, 0x1033, 0x1035, 0x1036, 0x1103,) self.blaslt_supported = os.path.exists(os.path.join(blaslt_tensile_libpath, f"Kernels.so-000-{name}.hsaco" if sys.platform == "win32" else f"extop_{name}.co")) + @property + def therock(self) -> str: + if (self.gfx_version & 0xFFF0) == 0x1100: + return "gfx110X-dgpu" + if self.gfx_version == 0x1151: + return "gfx1151" + if (self.gfx_version & 0xFFF0) == 0x1200: + return "gfx120X-all" + if (self.gfx_version & 0xFFF0) == 0x940: + return "gfx94X-dcgpu" + if self.gfx_version == 0x950: + return "gfx950-dcgpu" + raise RuntimeError(f"Unsupported GPU architecture: {self.name}") + def get_gfx_version(self) -> Union[str, None]: - if self.gfx_version >= 0x1101 and self.gfx_version < 0x1200: + if self.gfx_version >= 0x1100 and self.gfx_version < 0x1200: return "11.0.0" elif self.gfx_version != 0x1030 and self.gfx_version >= 0x1000 and self.gfx_version < 0x1100: # gfx1010 users had to override gfx version to 10.3.0 in Linux @@ -90,26 +119,22 @@ class Agent: return None -def get_version_torch() -> Union[str, None]: - version_ = None - try: - version_ = importlib.metadata.version("torch") - except importlib.metadata.PackageNotFoundError: - return None - if "+rocm" not in version_: # unofficial build, non-rocm torch. - return None - return version_.split("+rocm")[1] +def find() -> Union[Environment, None]: + try: # TheRock + import _rocm_sdk_core # pylint: disable=unused-import + return PythonPackageEnvironment() + except ImportError: + pass + # system-wide installation + hip_path = shutil.which("hipconfig") + if hip_path is not None: + return ROCmEnvironment(dirname(resolve_link(hip_path), 2)) -if sys.platform == "win32": - def find() -> Union[str, None]: - hip_path = shutil.which("hipconfig") - if hip_path is not None: - return dirname(resolve_link(hip_path), 2) - + if sys.platform == "win32": hip_path = os.environ.get("HIP_PATH", None) if hip_path is not None: - return hip_path + return ROCmEnvironment(hip_path) program_files = os.environ.get('ProgramFiles', r'C:\Program Files') hip_path = rf'{program_files}\AMD\ROCm' @@ -146,29 +171,111 @@ if sys.platform == "win32": if latest is None: return None - return os.path.join(hip_path, str(latest)) - - def get_version() -> str: # cannot just run hipconfig as it requires Perl installed on Windows. - return os.path.basename(path) or os.path.basename(os.path.dirname(path)) - - def get_agents() -> List[Agent]: - return [Agent(x.split(' ')[-1].strip()) for x in spawn("hipinfo", cwd=os.path.join(path, 'bin')).split("\n") if x.startswith('gcnArchName:')] - - is_wsl: bool = False - version_torch = None -else: - def find() -> Union[str, None]: - rocm_path = shutil.which("hipconfig") - if rocm_path is not None: - return dirname(resolve_link(rocm_path), 2) + return ROCmEnvironment(os.path.join(hip_path, str(latest))) + else: if not os.path.exists("/opt/rocm"): return None - return resolve_link("/opt/rocm") + return ROCmEnvironment(resolve_link("/opt/rocm")) - def get_version() -> str: - arr = spawn("hipconfig --version", cwd=os.path.join(path, 'bin')).split(".") - return f'{arr[0]}.{arr[1]}' if len(arr) >= 2 else None +def get_version() -> str: + if isinstance(environment, ROCmEnvironment): + # We don't load the hip library that will not be used by PyTorch. + if sys.platform == "win32": + # ROCm is system-wide installed. Assume the version is the folder name. (e.g. C:\Program Files\AMD\ROCm\6.4) + # hipconfig requires Perl + return os.path.basename(environment.path) or os.path.basename(os.path.dirname(environment.path)) + else: + arr = spawn("hipconfig --version", cwd=os.path.join(environment.path, 'bin')).split(".") + return f'{arr[0]}.{arr[1]}' if len(arr) >= 2 else None + else: + # If rocm-sdk package is installed, the hip library may be used by PyTorch. + ver = ctypes.c_int() + environment.hip.hipRuntimeGetVersion(ctypes.byref(ver)) + major = ver.value // 10000000 + minor = (ver.value // 100000) % 100 + #patch = version.value % 100000 + return f"{major}.{minor}" + + +def get_flash_attention_command(agent: Agent) -> str: + default = "git+https://github.com/ROCm/flash-attention" + if agent.gfx_version >= 0x1100 and agent.gfx_version < 0x1200 and os.environ.get("FLASH_ATTENTION_USE_TRITON_ROCM", "false").lower() != "true": + # use the navi_rotary_fix fork because the original doesn't support rotary_emb for transformers + # original: "git+https://github.com/ROCm/flash-attention@howiejay/navi_support" + default = "git+https://github.com/Disty0/flash-attention@navi_rotary_fix" + return "--no-build-isolation " + os.environ.get("FLASH_ATTENTION_PACKAGE", default) + + +if sys.platform == "win32": + def get_agents() -> List[Agent]: + if isinstance(environment, ROCmEnvironment): + out = spawn("amdgpu-arch", cwd=os.path.join(environment.path, 'bin')) + else: + # Assume that amdgpu-arch is in PATH (venv/Scripts/amdgpu-arch.exe) + out = spawn("amdgpu-arch") + out = out.strip() + return [Agent(x.split(' ')[-1].strip()) for x in out.split("\n")] + + def driver_get_agents() -> List[Agent]: + # unsafe and experimental feature + from modules import windows_hip_ffi + hip = windows_hip_ffi.HIP() + count = hip.get_device_count() + agents = [None] * count + for i in range(count): + prop = hip.get_device_properties(i) + name = prop.gcnArchName.decode('utf-8').strip('\x00') + agents[i] = Agent(name) + del hip + return agents + + def postinstall(): + import torch + if torch.version.hip is None: + os.environ.pop("ROCM_HOME", None) + os.environ.pop("ROCM_PATH", None) + paths = os.environ["PATH"].split(";") + paths_no_rocm = [] + for path_ in paths: + if "rocm" not in path_.lower(): + paths_no_rocm.append(path_) + os.environ["PATH"] = ";".join(paths_no_rocm) + return + + def rocm_init(): + try: + import torch + import numpy as np + + original_cholesky_ex = torch.linalg.cholesky_ex + @wraps(original_cholesky_ex) + def cholesky_ex(A: torch.Tensor, upper=False, check_errors=False, out=None) -> torch.return_types.linalg_cholesky_ex: + assert not check_errors + return_device = A.device + L = torch.from_numpy(np.linalg.cholesky(A.to("cpu").numpy(), upper=upper)).to(return_device) + info = torch.tensor(0, dtype=torch.int32, device=return_device) + if out is not None: + out[0].copy_(L) + out[1].copy_(info) + return torch.return_types.linalg_cholesky_ex((L, info), {}) + torch.linalg.cholesky_ex = cholesky_ex + + original_cholesky = torch.linalg.cholesky + @wraps(original_cholesky) + def cholesky(A: torch.Tensor, upper=False, out=None) -> torch.Tensor: + return_device = A.device + L = torch.from_numpy(np.linalg.cholesky(A.to("cpu").numpy(), upper=upper)).to(return_device) + if out is not None: + out.copy_(L) + return L + torch.linalg.cholesky = cholesky + except Exception as e: + return False, e + return True, None + + is_wsl: bool = False +else: def get_agents() -> List[Agent]: try: agents = spawn("rocm_agent_enumerator").split("\n") @@ -178,41 +285,32 @@ else: agents = [x.strip().split(" ")[-1] for x in agents if x.startswith(' Name:') and "CPU" not in x] return [Agent(x) for x in agents] - def load_hsa_runtime() -> None: - try: - # Preload stdc++ library. This will ignore Anaconda stdc++ library. - load_library_global("/lib/x86_64-linux-gnu/libstdc++.so.6") - # Use tcmalloc if possible. - load_library_global("/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4") - except OSError: - pass - # Preload HSA Runtime library. - load_library_global("/opt/rocm/lib/libhsa-runtime64.so") + def postinstall(): + if is_wsl: + try: + if shutil.which("conda") is not None: + # Preload stdc++ library. This will bypass Anaconda stdc++ library. + load_library_global("/lib/x86_64-linux-gnu/libstdc++.so.6") + # Preload rocr4wsl. The user don't have to replace the library file. + load_library_global("/opt/rocm/lib/libhsa-runtime64.so") + except OSError: + pass - def set_blaslt_enabled(enabled: bool) -> None: - if enabled: - os.environ["HIPBLASLT_TENSILE_LIBPATH"] = blaslt_tensile_libpath - else: - os.environ["TORCH_BLAS_PREFER_HIPBLASLT"] = "0" - - def get_blaslt_enabled() -> bool: - return version == version_torch and bool(int(os.environ.get("TORCH_BLAS_PREFER_HIPBLASLT", "1"))) - - def get_flash_attention_command(agent: Agent): - default = "git+https://github.com/ROCm/flash-attention" - if agent.gfx_version >= 0x1100 and agent.gfx_version < 0x1200 and os.environ.get("FLASH_ATTENTION_USE_TRITON_ROCM", "false").lower() != "true": - # use the navi_rotary_fix fork because the original doesn't support rotary_emb for transformers - # original: "git+https://github.com/ROCm/flash-attention@howiejay/navi_support" - default = "git+https://github.com/Disty0/flash-attention@navi_rotary_fix" - return "--no-build-isolation " + os.environ.get("FLASH_ATTENTION_PACKAGE", default) + def rocm_init(): + return True, None is_wsl: bool = os.environ.get('WSL_DISTRO_NAME', 'unknown' if spawn('wslpath -w /') else None) is not None - version_torch = get_version_torch() -path = find() +environment = None blaslt_tensile_libpath = "" is_installed = False version = None -if path is not None: - blaslt_tensile_libpath = os.environ.get("HIPBLASLT_TENSILE_LIBPATH", os.path.join(path, "bin" if sys.platform == "win32" else "lib", "hipblaslt", "library")) - is_installed = True - version = get_version() + +def refresh(): + global environment, blaslt_tensile_libpath, is_installed, version # pylint: disable=global-statement + environment = find() + if environment is not None: + if isinstance(environment, ROCmEnvironment): + blaslt_tensile_libpath = os.environ.get("HIPBLASLT_TENSILE_LIBPATH", os.path.join(environment.path, "bin" if sys.platform == "win32" else "lib", "hipblaslt", "library")) + is_installed = True + version = get_version() +refresh() diff --git a/modules/safe.py b/modules/safe.py index 21b8c08b2..82d4ae77a 100644 --- a/modules/safe.py +++ b/modules/safe.py @@ -21,8 +21,8 @@ def encode(*args): class RestrictedUnpickler(pickle.Unpickler): extra_handler = None - def persistent_load(self, saved_id): - assert saved_id[0] == 'storage' + def persistent_load(self, pid): + assert pid[0] == 'storage' try: return TypedStorage(_internal=True) except TypeError: diff --git a/modules/sd_checkpoint.py b/modules/sd_checkpoint.py index a1ea0d8ed..8857b3ccf 100644 --- a/modules/sd_checkpoint.py +++ b/modules/sd_checkpoint.py @@ -22,11 +22,12 @@ warn_once = False class CheckpointInfo: - def __init__(self, filename, sha=None): + def __init__(self, filename, sha=None, subfolder=None): self.name = None self.hash = sha self.filename = filename self.type = '' + self.subfolder = subfolder relname = filename app_path = os.path.abspath(paths.script_path) @@ -106,7 +107,7 @@ class CheckpointInfo: return self.shorthash def __str__(self): - return f"CheckpointInfo(name={self.name} filename={self.filename} hash={self.shorthash} type={self.type}" + return f'CheckpointInfo(name="{self.name}" filename="{self.filename}" hash={self.shorthash} type={self.type} title="{self.title}" path="{self.path}" subfolder="{self.subfolder}")' def setup_model(): @@ -196,13 +197,13 @@ def remove_hash(s): return re.sub(r'\s*\[.*?\]', '', s) -def get_closet_checkpoint_match(s: str) -> CheckpointInfo: +def get_closest_checkpoint_match(s: str) -> CheckpointInfo: + # direct hf url if s.startswith('https://huggingface.co/'): model_name = s.replace('https://huggingface.co/', '') checkpoint_info = CheckpointInfo(model_name) # create a virutal model info checkpoint_info.type = 'huggingface' return checkpoint_info - if s.startswith('huggingface/'): model_name = s.replace('huggingface/', '') checkpoint_info = CheckpointInfo(model_name) # create a virutal model info @@ -228,10 +229,27 @@ def get_closet_checkpoint_match(s: str) -> CheckpointInfo: # absolute path if s.endswith('.safetensors') and os.path.isfile(s): checkpoint_info = CheckpointInfo(s) + checkpoint_info.type = 'safetensors' + return checkpoint_info + + # reference search + ref = [(k, v) for k, v in shared.reference_models.items() if f"{v.get('path', '')}+{v.get('subfolder', '')}" == s] + if len(ref) == 0: + ref = [(k, v) for k, v in shared.reference_models.items() if v.get('path', '') == s] + if ref and len(ref) > 0: + _name, info = ref[0] + checkpoint_info = CheckpointInfo(s) + checkpoint_info.subfolder = info.get('subfolder', None) + checkpoint_info.type = 'reference' return checkpoint_info # huggingface search - if shared.opts.sd_checkpoint_autodownload and s.count('/') == 1: + if shared.opts.sd_checkpoint_autodownload and (s.count('/') == 1 or s.count('/') == 2): + if s.count('/') == 2: + subfolder = '/'.join(s.split('/')[2:]) # subfolder + s = '/'.join(s.split('/')[:2]) # only user/model + else: + subfolder = None modelloader.hf_login() found = modelloader.find_diffuser(s, full=True) if found is None: @@ -241,6 +259,8 @@ def get_closet_checkpoint_match(s: str) -> CheckpointInfo: if found is not None and len(found) == 1: checkpoint_info = CheckpointInfo(s) checkpoint_info.type = 'huggingface' + if subfolder is not None and len(subfolder) > 0: + checkpoint_info.subfolder = subfolder return checkpoint_info # civitai search @@ -274,7 +294,7 @@ def select_checkpoint(op='model', sd_model_checkpoint=None): model_checkpoint = sd_model_checkpoint or (shared.opts.data.get('sd_model_refiner', None) if op == 'refiner' else shared.opts.data.get('sd_model_checkpoint', None)) if model_checkpoint is None or model_checkpoint == 'None' or len(model_checkpoint) < 3: return None - checkpoint_info = get_closet_checkpoint_match(model_checkpoint) + checkpoint_info = get_closest_checkpoint_match(model_checkpoint) if checkpoint_info is not None: shared.log.info(f'Load {op}: select="{checkpoint_info.title if checkpoint_info is not None else None}"') return checkpoint_info @@ -289,7 +309,7 @@ def select_checkpoint(op='model', sd_model_checkpoint=None): return None if model_checkpoint is not None: if model_checkpoint != 'model.safetensors' and model_checkpoint != 'stabilityai/stable-diffusion-xl-base-1.0': - shared.log.info(f'Load {op}: search="{model_checkpoint}" not found') + shared.log.error(f'Load {op}: search="{model_checkpoint}" not found') else: shared.log.info("Selecting first available checkpoint") else: diff --git a/modules/sd_detect.py b/modules/sd_detect.py index b945a6457..0b2d3d19c 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -115,11 +115,17 @@ def guess_by_name(fn, current_guess): return 'Kandinsky 3.0' elif 'hunyuanimage' in fn.lower(): return 'HunyuanImage' + elif 'x-omni' in fn.lower(): + return 'X-Omni' + elif 'sdxl-turbo' in fn.lower() or 'stable-diffusion-xl' in fn.lower(): + return 'Stable Diffusion XL' return current_guess def guess_by_diffusers(fn, current_guess): exclude_by_name = ['ostris/Flex.2-preview'] # pipeline may be misleading + if not os.path.isdir(fn): + return current_guess, None index = os.path.join(fn, 'model_index.json') if os.path.exists(index) and os.path.isfile(index): index = shared.readfile(index, silent=True) @@ -132,9 +138,24 @@ def guess_by_diffusers(fn, current_guess): if pipeline is None: pipeline = cls if callable(pipeline): + is_quant = False + for folder in os.listdir(fn): + folder = os.path.join(fn, folder) + if is_quant: + break + if folder.endswith('quantization_config.json'): + is_quant = True + break + if os.path.isdir(folder): + for f in os.listdir(folder): + if f.endswith('quantization_config.json'): + is_quant = True + break pipelines = shared_items.get_pipelines() for k, v in pipelines.items(): if v is not None and v.__name__ == pipeline.__name__: + if is_quant: + k = f'{k} SDNQ' return k, v return current_guess, None @@ -193,6 +214,7 @@ def detect_pipeline(f: str, op: str = 'model'): def get_load_config(model_file, model_type, config_type='yaml'): + model_type = model_type.removesuffix(' SDNQ') if config_type == 'yaml': yaml = os.path.splitext(model_file)[0] + '.yaml' if os.path.exists(yaml): diff --git a/modules/sd_hijack_vae.py b/modules/sd_hijack_vae.py index edd8427e6..2563ccfed 100644 --- a/modules/sd_hijack_vae.py +++ b/modules/sd_hijack_vae.py @@ -18,7 +18,7 @@ def hijack_vae_decode(*args, **kwargs): latents = args[0].to(device=devices.device, dtype=shared.sd_model.vae.dtype) # upcast to vae dtype res = shared.sd_model.vae.orig_decode(latents, *args[1:], **kwargs) t1 = time.time() - shared.log.debug(f'Decode: vae={shared.sd_model.vae.__class__.__name__} slicing={getattr(shared.sd_model.vae, "use_slicing", None)} tiling={getattr(shared.sd_model.vae, "use_tiling", None)} latents={list(latents.shape)}:{latents.device}:{latents.dtype} time={t1-t0:.3f}') + shared.log.debug(f'Decode: vae={shared.sd_model.vae.__class__.__name__} slicing={getattr(shared.sd_model.vae, "use_slicing", None)} tiling={getattr(shared.sd_model.vae, "use_tiling", None)} latents={list(latents.shape)}:{latents.device} dtype={latents.dtype} time={t1-t0:.3f}') else: res = shared.sd_model.vae.orig_decode(*args, **kwargs) except Exception as e: diff --git a/modules/sd_models.py b/modules/sd_models.py index a251495c3..7c9ea0953 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -13,7 +13,7 @@ from installer import log from modules import timer, paths, shared, shared_items, modelloader, devices, script_callbacks, sd_vae, sd_unet, errors, sd_models_compile, sd_hijack_accelerate, sd_detect, model_quant, sd_hijack_te from modules.memstats import memory_stats from modules.modeldata import model_data -from modules.sd_checkpoint import CheckpointInfo, select_checkpoint, list_models, checkpoints_list, checkpoint_titles, get_closet_checkpoint_match, model_hash, update_model_hashes, setup_model, write_metadata, read_metadata_from_safetensors # pylint: disable=unused-import +from modules.sd_checkpoint import CheckpointInfo, select_checkpoint, list_models, checkpoints_list, checkpoint_titles, get_closest_checkpoint_match, model_hash, update_model_hashes, setup_model, write_metadata, read_metadata_from_safetensors # pylint: disable=unused-import from modules.sd_offload import disable_offload, set_diffuser_offload, apply_balanced_offload, set_accelerate # pylint: disable=unused-import from modules.sd_models_utils import NoWatermark, get_signature, get_call, path_to_repo, patch_diffuser_config, convert_to_faketensors, read_state_dict, get_state_dict_from_checkpoint, apply_function_to_model # pylint: disable=unused-import @@ -28,6 +28,7 @@ debug_move = log.trace if os.environ.get('SD_MOVE_DEBUG', None) is not None else debug_load = os.environ.get('SD_LOAD_DEBUG', None) debug_process = log.trace if os.environ.get('SD_PROCESS_DEBUG', None) is not None else lambda *args, **kwargs: None diffusers_version = int(diffusers.__version__.split('.')[1]) +get_closet_checkpoint_match = get_closest_checkpoint_match # legacy compatibility checkpoint_tiles = checkpoint_titles # legacy compatibility allow_post_quant = None pipe_switch_task_exclude = [ @@ -36,22 +37,19 @@ pipe_switch_task_exclude = [ 'FluxFillPipeline', 'InstantIRPipeline', 'LTXConditionPipeline', - 'OmniGenPipeline', - 'OmniGen2Pipeline', + 'OmniGenPipeline', 'OmniGen2Pipeline', 'PhotoMakerStableDiffusionXLPipeline', 'PixelSmithXLPipeline', 'StableDiffusion3ControlNetPipeline', - 'StableDiffusionAdapterPipeline', 'StableDiffusionAdapterPipeline', 'StableDiffusionXLAdapterPipeline', 'StableDiffusionControlNetXSPipeline', 'StableDiffusionXLControlNetXSPipeline', 'StableDiffusionReferencePipeline', 'StableDiffusionXLInstantIDPipeline', + 'XOmniPipeline', ] i2i_pipes = [ - 'LEditsPPPipelineStableDiffusion', - 'LEditsPPPipelineStableDiffusionXL', - 'OmniGenPipeline', - 'OmniGen2Pipeline', + 'LEditsPPPipelineStableDiffusion', 'LEditsPPPipelineStableDiffusionXL', + 'OmniGenPipeline', 'OmniGen2Pipeline', 'StableDiffusionAdapterPipeline', 'StableDiffusionXLAdapterPipeline', 'StableDiffusionControlNetXSPipeline', 'StableDiffusionXLControlNetXSPipeline', ] @@ -398,6 +396,10 @@ def load_diffuser_force(model_type, checkpoint_info, diffusers_load_config, op=' from pipelines.model_hyimage import load_hyimage sd_model = load_hyimage(checkpoint_info, diffusers_load_config) # pylint: disable=assignment-from-none allow_post_quant = False + elif model_type in ['X-Omni']: + from pipelines.model_xomni import load_xomni + sd_model = load_xomni(checkpoint_info, diffusers_load_config) # pylint: disable=assignment-from-none + allow_post_quant = False except Exception as e: shared.log.error(f'Load {op}: path="{checkpoint_info.path}" {e}') if debug_load: @@ -411,65 +413,70 @@ def load_diffuser_folder(model_type, pipeline, checkpoint_info, diffusers_load_c files = shared.walk_files(checkpoint_info.path, ['.safetensors', '.bin', '.ckpt']) if 'variant' not in diffusers_load_config and any('diffusion_pytorch_model.fp16' in f for f in files): # deal with diffusers lack of variant fallback when loading diffusers_load_config['variant'] = 'fp16' - if (model_type is not None) and (pipeline is not None) and ('ONNX' in model_type): # forced pipeline - try: - sd_model = pipeline.from_pretrained(checkpoint_info.path) - except Exception as e: - shared.log.error(f'Load {op}: type=ONNX path="{checkpoint_info.path}" {e}') - if debug_load: - errors.display(e, 'Load') - return None - else: - err1, err2, err3 = None, None, None - if os.path.exists(checkpoint_info.path) and os.path.isdir(checkpoint_info.path): - if os.path.exists(os.path.join(checkpoint_info.path, 'unet', 'diffusion_pytorch_model.bin')): - shared.log.debug(f'Load {op}: type=pickle') - diffusers_load_config['use_safetensors'] = False + + err0, err1, err2, err3 = None, None, None, None + if os.path.exists(checkpoint_info.path) and os.path.isdir(checkpoint_info.path): + if os.path.exists(os.path.join(checkpoint_info.path, 'unet', 'diffusion_pytorch_model.bin')): + shared.log.debug(f'Load {op}: type=pickle') + diffusers_load_config['use_safetensors'] = False + if debug_load: + shared.log.debug(f'Load {op}: args={diffusers_load_config}') + + try: #0 - using detected model type and pipeline + if (model_type is not None) and (pipeline is not None): + sd_model = pipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model.model_type = sd_model.__class__.__name__ + except Exception as e: + err0 = e if debug_load: - shared.log.debug(f'Load {op}: args={diffusers_load_config}') - try: # 1 - autopipeline, best choice but not all pipelines are available - try: + errors.display(e, 'Load Detected') + + try: # 1 - autopipeline, best choice but not all pipelines are available + try: + if err0 is not None: sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) sd_model.model_type = sd_model.__class__.__name__ - except ValueError as e: - if 'no variant default' in str(e): - shared.log.warning(f'Load {op}: variant={diffusers_load_config["variant"]} model="{checkpoint_info.path}" using default variant') - diffusers_load_config.pop('variant', None) - sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) - sd_model.model_type = sd_model.__class__.__name__ - elif 'safetensors found in directory' in str(err1): - shared.log.warning(f'Load {op}: type=pickle') - diffusers_load_config['use_safetensors'] = False - sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) - sd_model.model_type = sd_model.__class__.__name__ - else: - raise ValueError from e # reraise - except Exception as e: - err1 = e - if debug_load: - errors.display(e, 'Load AutoPipeline') - # shared.log.error(f'AutoPipeline: {e}') - try: # 2 - diffusion pipeline, works for most non-linked pipelines - if err1 is not None: - sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + except ValueError as e: + if 'no variant default' in str(e): + shared.log.warning(f'Load {op}: variant={diffusers_load_config["variant"]} model="{checkpoint_info.path}" using default variant') + diffusers_load_config.pop('variant', None) + sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) sd_model.model_type = sd_model.__class__.__name__ - except Exception as e: - err2 = e - if debug_load: - errors.display(e, "Load DiffusionPipeline") - # shared.log.error(f'DiffusionPipeline: {e}') - try: # 3 - try basic pipeline just in case - if err2 is not None: - sd_model = diffusers.StableDiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + elif 'safetensors found in directory' in str(err1): + shared.log.warning(f'Load {op}: type=pickle') + diffusers_load_config['use_safetensors'] = False + sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) sd_model.model_type = sd_model.__class__.__name__ - except Exception as e: - err3 = e # ignore last error - shared.log.error(f"StableDiffusionPipeline: {e}") - if debug_load: - errors.display(e, "Load StableDiffusionPipeline") - if err3 is not None: - shared.log.error(f'Load {op}: {checkpoint_info.path} auto={err1} diffusion={err2}') - return None + else: + raise ValueError from e # reraise + except Exception as e: + err1 = e + if debug_load: + errors.display(e, 'Load AutoPipeline') + + try: # 2 - diffusion pipeline, works for most non-linked pipelines + if err1 is not None: + sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model.model_type = sd_model.__class__.__name__ + except Exception as e: + err2 = e + if debug_load: + errors.display(e, "Load DiffusionPipeline") + + try: # 3 - try basic pipeline just in case + if err2 is not None: + sd_model = diffusers.StableDiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model.model_type = sd_model.__class__.__name__ + except Exception as e: + err3 = e # ignore last error + shared.log.error(f"StableDiffusionPipeline: {e}") + if debug_load: + errors.display(e, "Load StableDiffusionPipeline") + + if err3 is not None: + shared.log.error(f'Load {op}: {checkpoint_info.path} detected={err0} auto={err1} diffusion={err2} base={err3}') + return None + return sd_model @@ -521,7 +528,7 @@ def load_diffuser_file(model_type, pipeline, checkpoint_info, diffusers_load_con diffusers_load_config.pop('local_files_only', None) shared.log.debug(f'Setting {op}: pipeline={sd_model.__class__.__name__} config={diffusers_load_config}') # pylint: disable=protected-access except Exception as e: - shared.log.error(f'Load {op}: file="{checkpoint_info.path}" pipeline={shared.opts.diffusers_pipeline}/{sd_model.__class__.__name__} config={diffusers_load_config} {e}') + shared.log.error(f'Load {op}: file="{checkpoint_info.path}" pipeline={shared.opts.diffusers_pipeline} config={diffusers_load_config} {e}') if 'Weights for this component appear to be missing in the checkpoint' in str(e): shared.log.error(f'Load {op}: file="{checkpoint_info.path}" is not a complete model') else: @@ -530,6 +537,35 @@ def load_diffuser_file(model_type, pipeline, checkpoint_info, diffusers_load_con return sd_model +def load_sdnq_model(checkpoint_info, pipeline, diffusers_load_config, op): + from modules import sdnq + modules = {} + for module_name in os.listdir(checkpoint_info.path): + quantization_config_path = os.path.join(checkpoint_info.path, module_name, 'quantization_config.json') + if not os.path.exists(quantization_config_path): + continue + model_name = os.path.join(checkpoint_info.path, module_name) + quantization_config = shared.readfile(quantization_config_path, silent=True) + shared.log.debug(f'Load {op}: model="{checkpoint_info.name}" module="{module_name}" direct={shared.opts.diffusers_to_gpu} prequant=sdnq') + try: + modules[module_name] = sdnq.load_sdnq_model( + model_path=model_name, + quantization_config=quantization_config, + device=devices.device if shared.opts.diffusers_to_gpu else devices.cpu, + dtype=devices.dtype, + ) + except Exception as e: + shared.log.error(f'Load {op}: model="{checkpoint_info.name}" module="{module_name}" {e}') + errors.display(e, 'Load') + sd_model = pipeline.from_pretrained( + checkpoint_info.path, + cache_dir=shared.opts.diffusers_dir, + **modules, + **diffusers_load_config, + ) + return sd_model + + def set_overrides(sd_model, checkpoint_info): checkpoint_info_name = checkpoint_info.name.lower() if 'bigaspv25' in checkpoint_info_name or ('flow' in checkpoint_info_name and 'flower' not in checkpoint_info_name): @@ -646,6 +682,13 @@ def load_diffuser(checkpoint_info=None, op='model', revision=None): # pylint: di shared.log.error(f'Load {op}: type="{model_type}" pipeline="{pipeline}" not loaded') return + # load sdnq-prequantized model + if sd_model is None: + if model_type.endswith('SDNQ'): + sd_model = load_sdnq_model(checkpoint_info, pipeline, diffusers_load_config, op) + allow_post_quant = False + model_type = model_type.replace(' SDNQ', '') + # load from hf folder-style if sd_model is None: if os.path.isdir(checkpoint_info.path) or checkpoint_info.type == 'huggingface' or checkpoint_info.type == 'transformer': @@ -667,7 +710,7 @@ def load_diffuser(checkpoint_info=None, op='model', revision=None): # pylint: di sd_model.scheduler.name = 'DDIM' if hasattr(sd_model, "unet") and model_type not in ['Stable Cascade']: # others calls load_diffuser again - sd_unet.load_unet(sd_model) + sd_unet.load_unet(sd_model, checkpoint_info.path) add_noise_pred_to_diffusers_callback(sd_model) @@ -738,6 +781,7 @@ class DiffusersTaskType(Enum): IMAGE_2_IMAGE = 2 INPAINTING = 3 INSTRUCT = 4 + MODULAR = 5 def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType: @@ -748,6 +792,8 @@ def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType: return DiffusersTaskType.IMAGE_2_IMAGE elif 'Instruct' in cls: return DiffusersTaskType.INSTRUCT + elif 'Modular' in cls: + return DiffusersTaskType.MODULAR elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING.values(): return DiffusersTaskType.IMAGE_2_IMAGE elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING.values(): @@ -945,6 +991,9 @@ def set_diffuser_pipe(pipe, new_pipe_type): if get_diffusers_task(pipe) == new_pipe_type: return pipe + if get_diffusers_task(pipe) == DiffusersTaskType.MODULAR: + return pipe + # skip specific pipelines cls = pipe.__class__.__name__ if cls in pipe_switch_task_exclude: @@ -1028,8 +1077,14 @@ def set_diffusers_attention(pipe, quiet:bool=False): if attention is None: return # other models uses their own attention processor - if pipe.__class__.__name__.startswith("StableDiffusion") and hasattr(pipe, "unet"): - pipe.unet.set_attn_processor(attention) + if pipe.__class__.__name__.startswith("StableDiffusion") and getattr(pipe, "unet", None) is not None and hasattr(pipe.unet, "set_attn_processor"): + try: + pipe.unet.set_attn_processor(attention) + except Exception as e: + if 'Nunchaku' in pipe.unet.__class__.__name__: + pass + else: + shared.log.error(f"Attention: {name if name is not None else attention.__class__.__name__} pipe={pipe.__class__.__name__} {e}") elif not quiet: shared.log.warning(f"Attention: {name if name is not None else attention.__class__.__name__} is not compatible with {pipe.__class__.__name__}") @@ -1053,17 +1108,19 @@ def set_diffusers_attention(pipe, quiet:bool=False): pipe.enable_xformers_memory_efficient_attention() else: shared.log.warning(f"Attention: xFormers is not compatible with {pipe.__class__.__name__}") - elif shared.opts.cross_attention_optimization == "Split attention": - if hasattr(pipe, "enable_attention_slicing"): - pipe.enable_attention_slicing() - else: - shared.log.warning(f"Attention: Split attention is not compatible with {pipe.__class__.__name__}") elif shared.opts.cross_attention_optimization == "Batch matrix-matrix": set_attn(pipe, p.AttnProcessor(), name="Batch matrix-matrix") elif shared.opts.cross_attention_optimization == "Dynamic Attention BMM": from modules.sd_hijack_dynamic_atten import DynamicAttnProcessorBMM set_attn(pipe, DynamicAttnProcessorBMM(), name="Dynamic Attention BMM") + if shared.opts.attention_slicing != "Default" and hasattr(pipe, "enable_attention_slicing") and hasattr(pipe, "disable_attention_slicing"): + if shared.opts.attention_slicing: + pipe.enable_attention_slicing() + else: + pipe.disable_attention_slicing() + shared.log.debug(f"Attention: slicing={shared.opts.attention_slicing}") + pipe.current_attn_name = shared.opts.cross_attention_optimization @@ -1201,10 +1258,41 @@ def hf_auth_check(checkpoint_info, force:bool=False): return True except Exception: pass + repo_id = path_to_repo(checkpoint_info) try: login = modelloader.hf_login() - repo_id = path_to_repo(checkpoint_info) return hf.auth_check(repo_id) except Exception as e: shared.log.error(f'Load model: repo="{repo_id}" login={login} {e}') return False + + +def save_model(name: str, path: str = None, shard: str = None, overwrite: bool = False): + if (name is None) or len(name.strip()) == 0: + shared.log.error('Save model: invalid model name') + return 'Invalid model name' + if not shared.sd_loaded: + shared.log.error('Save model: model not loaded') + return 'Model not loaded' + from modules.sdnq import save_sdnq_model + if path is None: + path = shared.opts.diffusers_dir + model_name = os.path.join(path.strip(), name.strip()) + if os.path.exists(model_name) and not overwrite: + shared.log.error(f'Save model: path="{model_name}" exists') + return f'Path exists: {model_name}' + try: + t0 = time.time() + save_sdnq_model( + model=shared.sd_model, + model_path=model_name, + max_shard_size=shard, + is_pipeline=True, + ) + t1 = time.time() + shared.log.info(f'Save model: path="{model_name}" cls={shared.sd_model.__class__.__name__} time={t1 - t0:.2f}') + return f'Saved: {model_name}' + except Exception as e: + shared.log.error(f'Save model: path="{model_name}" {e}') + errors.display(e, 'Save model') + return f'Error: {e}' diff --git a/modules/sd_models_utils.py b/modules/sd_models_utils.py index 0d201779c..05f44ef6c 100644 --- a/modules/sd_models_utils.py +++ b/modules/sd_models_utils.py @@ -8,7 +8,7 @@ import torch import safetensors.torch from modules import paths, shared, errors -from modules.sd_checkpoint import CheckpointInfo, select_checkpoint, list_models, checkpoints_list, checkpoint_titles, get_closet_checkpoint_match, model_hash, update_model_hashes, setup_model, write_metadata, read_metadata_from_safetensors # pylint: disable=unused-import +from modules.sd_checkpoint import CheckpointInfo, select_checkpoint, list_models, checkpoints_list, checkpoint_titles, get_closest_checkpoint_match, model_hash, update_model_hashes, setup_model, write_metadata, read_metadata_from_safetensors # pylint: disable=unused-import from modules.sd_offload import disable_offload, set_diffuser_offload, apply_balanced_offload, set_accelerate # pylint: disable=unused-import @@ -46,6 +46,8 @@ def path_to_repo(checkpoint_info): repo_id = repo_id.replace('--', '/') if repo_id.count('/') != 1: shared.log.warning(f'Model: repo="{repo_id}" repository not recognized') + if '+' in repo_id: + repo_id = repo_id.split('+')[0] return repo_id @@ -162,6 +164,8 @@ def apply_function_to_model(sd_model, function, options, op=None): sd_model.unet = function(sd_model.unet, op="unet", sd_model=sd_model) if hasattr(sd_model, 'transformer') and hasattr(sd_model.transformer, 'config'): sd_model.transformer = function(sd_model.transformer, op="transformer", sd_model=sd_model) + if hasattr(sd_model, 'dit') and hasattr(sd_model.dit, 'config'): + sd_model.dit = function(sd_model.dit, op="dit", sd_model=sd_model) if hasattr(sd_model, 'transformer_2') and hasattr(sd_model.transformer_2, 'config'): sd_model.transformer_2 = function(sd_model.transformer_2, op="transformer_2", sd_model=sd_model) if hasattr(sd_model, 'transformer_3') and hasattr(sd_model.transformer_3, 'config'): diff --git a/modules/sd_offload.py b/modules/sd_offload.py index bf951e2c5..0d9defd5d 100644 --- a/modules/sd_offload.py +++ b/modules/sd_offload.py @@ -12,12 +12,15 @@ from modules.timer import process as process_timer debug = os.environ.get('SD_MOVE_DEBUG', None) is not None +verbose = os.environ.get('SD_MOVE_VERBOSE', None) is not None debug_move = log.trace if debug else lambda *args, **kwargs: None -offload_warn = ['sc', 'sd3', 'f1', 'h1', 'hunyuandit', 'auraflow', 'omnigen', 'omnigen2', 'cogview4', 'cosmos', 'chroma'] +offload_warn = ['sc', 'sd3', 'f1', 'h1', 'hunyuandit', 'auraflow', 'omnigen', 'omnigen2', 'cogview4', 'cosmos', 'chroma', 'x-omni'] offload_post = ['h1'] offload_hook_instance = None balanced_offload_exclude = ['CogView4Pipeline', 'MeissonicPipeline'] +no_split_module_classes = ["Linear", "Conv1d", "Conv2d", "Conv3d", "ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d"] accelerate_dtype_byte_size = None +move_stream = None def dtype_byte_size(dtype: torch.dtype): @@ -176,7 +179,7 @@ class OffloadHook(accelerate.hooks.ModelHook): self.last_post = None self.last_cls = None gpu = f'{(shared.gpu_memory * shared.opts.diffusers_offload_min_gpu_memory):.2f}-{(shared.gpu_memory * shared.opts.diffusers_offload_max_gpu_memory):.2f}:{shared.gpu_memory:.2f}' - shared.log.info(f'Offload: type=balanced op=init watermark={self.min_watermark}-{self.max_watermark} gpu={gpu} cpu={shared.cpu_memory:.3f} limit={shared.opts.cuda_mem_fraction:.2f} always={self.offload_always} never={self.offload_never} pre={shared.opts.diffusers_offload_pre}') + shared.log.info(f'Offload: type=balanced op=init watermark={self.min_watermark}-{self.max_watermark} gpu={gpu} cpu={shared.cpu_memory:.3f} limit={shared.opts.cuda_mem_fraction:.2f} always={self.offload_always} never={self.offload_never} pre={shared.opts.diffusers_offload_pre} streams={shared.opts.diffusers_offload_streams}') self.validate() super().__init__() @@ -206,48 +209,56 @@ class OffloadHook(accelerate.hooks.ModelHook): return False if hasattr(module, 'nets') and any(hasattr(n, "offload_never") for n in module.nets): return False + if shared.sd_model_type.lower() in [m.lower().strip() for m in re.split(r'[ ,]+', shared.opts.models_not_to_offload)]: + return False return True def pre_forward(self, module, *args, **kwargs): _id = id(module) - if (self.last_pre != _id) and (module.__class__.__name__ != self.last_cls) and self.offload_allowed(module): # offload every other module first time when new module starts pre-forward + + do_offload = (self.last_pre != _id) or (module.__class__.__name__ != self.last_cls) + + if do_offload and self.offload_allowed(module): # offload every other module first time when new module starts pre-forward if shared.opts.diffusers_offload_pre: + t0 = time.time() debug_move(f'Offload: type=balanced op=pre module={module.__class__.__name__}') for pipe in get_pipe_variants(): for module_name in get_module_names(pipe): module_instance = getattr(pipe, module_name, None) module_cls = module_instance.__class__.__name__ - if (_id != id(module_instance)) and (module_cls not in self.offload_never) and (not devices.same_device(module_instance.device, devices.cpu)): + if (module_instance is not None) and (_id != id(module_instance)) and (module_cls not in self.offload_never) and (not devices.same_device(module_instance.device, devices.cpu)): apply_balanced_offload_to_module(module_instance, op='pre') self.last_cls = module.__class__.__name__ - self.last_pre = _id + process_timer.add('offload', time.time() - t0) if not devices.same_device(module.device, devices.device): # move-to-device + t0 = time.time() device_index = torch.device(devices.device).index if device_index is None: device_index = 0 max_memory = { device_index: self.gpu, "cpu": self.cpu } device_map = getattr(module, "balanced_offload_device_map", None) - if device_map is None or max_memory != getattr(module, "balanced_offload_max_memory", None): - device_map = accelerate.infer_auto_device_map(module, max_memory=max_memory) + if (device_map is None) or (max_memory != getattr(module, "balanced_offload_max_memory", None)): + device_map = accelerate.infer_auto_device_map(module, max_memory=max_memory, no_split_module_classes=no_split_module_classes, verbose=verbose) offload_dir = getattr(module, "offload_dir", os.path.join(shared.opts.accelerate_offload_path, module.__class__.__name__)) if devices.backend == "directml": - keys = device_map.keys() - for v in keys: - if isinstance(device_map[v], int): - device_map[v] = f"{devices.device.type}:{device_map[v]}" # int implies CUDA or XPU device, but it will break DirectML backend so we add type + for k, v in device_map.items(): + if isinstance(v, int): + device_map[k] = f"{devices.device.type}:{v}" # int implies CUDA or XPU device, but it will break DirectML backend so we add type if device_map is not None: module = accelerate.dispatch_model(module, device_map=device_map, offload_dir=offload_dir) module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access module.balanced_offload_device_map = device_map module.balanced_offload_max_memory = max_memory + process_timer.add('onload', time.time() - t0) if debug: - for pipe in get_pipe_variants(): + for _i, pipe in enumerate(get_pipe_variants()): for module_name in get_module_names(pipe): module_instance = getattr(pipe, module_name, None) - shared.log.trace(f'Offload: type=balanced op=pre:status module={module_instance.__class__.__name__} device={module_instance.device} dtype={module_instance.dtype}') + shared.log.trace(f'Offload: type=balanced op=pre:status forward={module.__class__.__name__} module={module_name} class={module_instance.__class__.__name__} pipe={_i} device={module_instance.device} dtype={module_instance.dtype}') + self.last_pre = _id return args, kwargs def post_forward(self, module, output): @@ -263,7 +274,10 @@ class OffloadHook(accelerate.hooks.ModelHook): def get_pipe_variants(pipe=None): if pipe is None: - pipe = shared.sd_model + if shared.sd_loaded: + pipe = shared.sd_model + else: + return [pipe] variants = [pipe] if hasattr(pipe, "pipe"): variants.append(pipe.pipe) @@ -276,13 +290,17 @@ def get_pipe_variants(pipe=None): def get_module_names(pipe=None, exclude=[]): if pipe is None: - pipe = shared.sd_model + if shared.sd_loaded: + pipe = shared.sd_model + else: + return [] if hasattr(pipe, "_internal_dict"): modules_names = pipe._internal_dict.keys() # pylint: disable=protected-access else: modules_names = get_signature(pipe).keys() modules_names = [m for m in modules_names if m not in exclude and not m.startswith('_')] modules_names = [m for m in modules_names if isinstance(getattr(pipe, m, None), torch.nn.Module)] + modules_names = sorted(set(modules_names)) return modules_names @@ -308,6 +326,17 @@ def get_module_sizes(pipe=None, exclude=[]): def move_module_to_cpu(module, op='unk', force:bool=False): + def do_move(module): + if shared.opts.diffusers_offload_streams: + global move_stream # pylint: disable=global-statement + if move_stream is None: + move_stream = torch.cuda.Stream(device=devices.device) + with torch.cuda.stream(move_stream): + module = module.to(devices.cpu) + else: + module = module.to(devices.cpu) + return module + try: module_name = getattr(module, "module_name", module.__class__.__name__) module_size = offload_hook_instance.offload_map.get(module_name, offload_hook_instance.model_size()) @@ -318,17 +347,17 @@ def move_module_to_cpu(module, op='unk', force:bool=False): op = f'{op}:skip' if force: op = f'{op}:force' - module = module.to(devices.cpu) + module = do_move(module) used_gpu -= module_size elif module_cls in offload_hook_instance.offload_never: op = f'{op}:never' elif module_cls in offload_hook_instance.offload_always: op = f'{op}:always' - module = module.to(devices.cpu) + module = do_move(module) used_gpu -= module_size elif perc_gpu > shared.opts.diffusers_offload_min_gpu_memory: op = f'{op}:mem' - module = module.to(devices.cpu) + module = do_move(module) used_gpu -= module_size if debug: quant = getattr(module, "quantization_method", None) @@ -412,6 +441,7 @@ def apply_balanced_offload(sd_model=None, exclude=[]): module.offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) apply_balanced_offload_to_module(module, op='apply') report_model_stats(module_name, module) + set_accelerate(sd_model) t = time.time() - t0 process_timer.add('offload', t) diff --git a/modules/sd_unet.py b/modules/sd_unet.py index 2bd1741f4..6d3d6c936 100644 --- a/modules/sd_unet.py +++ b/modules/sd_unet.py @@ -1,5 +1,5 @@ import os -from modules import shared, devices, files_cache, sd_models +from modules import shared, devices, files_cache, sd_models, model_quant unet_dict = {} @@ -8,22 +8,55 @@ failed_unet = [] debug = os.environ.get('SD_LOAD_DEBUG', None) is not None -dit_models = ['Flux', 'StableDiffusion3', 'HiDream', 'Lumina2', 'Chroma', 'Wan'] +dit_models = ['Flux', 'StableDiffusion3', 'HiDream', 'Lumina2', 'Chroma', 'Wan', 'Qwen'] -def load_unet(model): +def load_unet_sdxl_nunchaku(repo_id): + try: + from nunchaku.models.unets.unet_sdxl import NunchakuSDXLUNet2DConditionModel + except Exception: + shared.log.error(f'Load module: quant=Nunchaku module=unet repo="{repo_id}" low nunchaku version') + return None + if 'turbo' in repo_id.lower(): + nunchaku_repo = 'nunchaku-tech/nunchaku-sdxl-turbo/svdq-int4_r32-sdxl-turbo.safetensors' + else: + nunchaku_repo = 'nunchaku-tech/nunchaku-sdxl/svdq-int4_r32-sdxl.safetensors' + + shared.log.debug(f'Load module: quant=Nunchaku module=unet repo="{nunchaku_repo}" offload={shared.opts.nunchaku_offload}') + unet = NunchakuSDXLUNet2DConditionModel.from_pretrained( + nunchaku_repo, + offload=shared.opts.nunchaku_offload, + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + ) + unet.quantization_method = 'SVDQuant' + return unet + + +def load_unet(model, repo_id:str=None): global loaded_unet # pylint: disable=global-statement + + if ("StableDiffusionXLPipeline" in model.__class__.__name__) and (('stable-diffusion-xl-base' in repo_id) or ('sdxl-turbo' in repo_id)): + if model_quant.check_nunchaku('Model'): + unet = load_unet_sdxl_nunchaku(repo_id) + if unet is not None: + model.unet = unet + return + if shared.opts.sd_unet == 'Default' or shared.opts.sd_unet == 'None': return + if shared.opts.sd_unet not in list(unet_dict): shared.log.error(f'Load module: type=UNet not found: {shared.opts.sd_unet}') return + config_file = os.path.splitext(unet_dict[shared.opts.sd_unet])[0] + '.json' if os.path.exists(config_file): config = shared.readfile(config_file) else: config = None config_file = 'default' + try: if shared.opts.sd_unet == loaded_unet or shared.opts.sd_unet in failed_unet: pass diff --git a/modules/sd_vae_taesd.py b/modules/sd_vae_taesd.py index aed995f1b..510215855 100644 --- a/modules/sd_vae_taesd.py +++ b/modules/sd_vae_taesd.py @@ -79,6 +79,7 @@ def get_model(model_type = 'decoder', variant = None): uri += '/tae' + model_cls + '_' + model_type + '.pth' try: torch.hub.download_url_to_file(uri, fn) + shared.log.print() # new line shared.log.info(f'Decode: type="taesd" variant="{variant}": uri="{uri}" fn="{fn}" download') except Exception as e: warn_once(f'download uri={uri} {e}', variant=variant) @@ -86,7 +87,8 @@ def get_model(model_type = 'decoder', variant = None): prev_cls = model_cls prev_type = model_type prev_model = variant - shared.log.debug(f'Decode: type="taesd" variant="{variant}" fn="{fn}" load') + shared.log.print() # new line + shared.log.debug(f'Decode: type="taesd" variant="{variant}" fn="{fn}" layers={shared.opts.taesd_layers} load') vae = None if 'TAE HunyuanVideo' in variant: from modules.taesd.taehv import TAEHV diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index a7829d44d..c67bf0db7 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -1,552 +1,14 @@ -# pylint: disable=redefined-builtin,no-member,protected-access - -from typing import Any, Dict, List, Tuple, Optional, Union -from dataclasses import dataclass -from enum import Enum - -import re -import torch -from diffusers.quantizers.base import DiffusersQuantizer -from diffusers.quantizers.quantization_config import QuantizationConfigMixin -from diffusers.utils import get_module_from_name -from modules import devices, shared - -from .common import dtype_dict, use_tensorwise_fp8_matmul, allowed_types, conv_types, conv_transpose_types -from .dequantizer import dequantizer_dict -from .forward import get_forward_func +from .quantizer import QuantizationMethod, SDNQConfig, SDNQQuantizer, sdnq_post_load_quant, apply_sdnq_to_module, sdnq_quantize_layer +from .loader import save_sdnq_model, load_sdnq_model -class QuantizationMethod(str, Enum): - SDNQ = "sdnq" - - -def get_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: Union[int, List[int]], weights_dtype: str) -> Tuple[torch.FloatTensor, torch.FloatTensor]: - zero_point = torch.amin(weight, dim=reduction_axes, keepdims=True) - scale = torch.amax(weight, dim=reduction_axes, keepdims=True).sub_(zero_point).div_(dtype_dict[weights_dtype]["max"] - dtype_dict[weights_dtype]["min"]) - if dtype_dict[weights_dtype]["min"] != 0: - zero_point.sub_(torch.mul(scale, dtype_dict[weights_dtype]["min"])) - return scale, zero_point - - -def get_scale_symmetric(weight: torch.FloatTensor, reduction_axes: Union[int, List[int]], weights_dtype: str) -> torch.FloatTensor: - return torch.amax(weight.abs(), dim=reduction_axes, keepdims=True).div_(dtype_dict[weights_dtype]["max"]) - - -def quantize_weight(weight: torch.FloatTensor, reduction_axes: Union[int, List[int]], weights_dtype: str) -> Tuple[torch.Tensor, torch.FloatTensor, torch.FloatTensor]: - if dtype_dict[weights_dtype]["is_unsigned"]: - scale, zero_point = get_scale_asymmetric(weight, reduction_axes, weights_dtype) - quantized_weight = torch.sub(weight, zero_point).div_(scale) - else: - scale = get_scale_symmetric(weight, reduction_axes, weights_dtype) - quantized_weight = torch.div(weight, scale) - zero_point = None - if dtype_dict[weights_dtype]["is_integer"]: - quantized_weight.round_() - else: - quantized_weight.nan_to_num_() - quantized_weight = quantized_weight.clamp_(dtype_dict[weights_dtype]["min"], dtype_dict[weights_dtype]["max"]).to(dtype_dict[weights_dtype]["torch_dtype"]) - return quantized_weight, scale, zero_point - - -@devices.inference_context() -def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, dequantize_fp32=False, non_blocking=False, quantization_device=None, return_device=None, param_name=None): # pylint: disable=unused-argument - layer_class_name = layer.__class__.__name__ - if layer_class_name in allowed_types: - num_of_groups = 1 - is_conv_type = False - is_conv_transpose_type = False - is_linear_type = False - result_shape = None - original_shape = layer.weight.shape - if torch_dtype is None: - torch_dtype = devices.dtype - - if layer_class_name in conv_types: - if not quant_conv: - return layer - if dtype_dict[weights_dtype]["num_bits"] < 4: - weights_dtype = "uint4" - is_conv_type = True - reduction_axes = 1 - output_channel_size, channel_size = layer.weight.shape[:2] - group_channel_size = channel_size // layer.groups - use_quantized_matmul = False - if use_quantized_matmul_conv: - use_quantized_matmul = group_channel_size >= 32 and output_channel_size >= 32 - if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: - use_quantized_matmul = output_channel_size % 16 == 0 and group_channel_size % 16 == 0 - elif layer_class_name in conv_transpose_types: - if not quant_conv: - return layer - if dtype_dict[weights_dtype]["num_bits"] < 4: - weights_dtype = "uint4" - is_conv_transpose_type = True - reduction_axes = 0 - channel_size, output_channel_size = layer.weight.shape[:2] - use_quantized_matmul = False - else: - is_linear_type = True - reduction_axes = -1 - try: - output_channel_size, channel_size = layer.weight.shape - except Exception as e: - raise ValueError(f"SDNQ: param_name={param_name} layer_class_name={layer_class_name} layer_weight_shape={layer.weight.shape} weights_dtype={weights_dtype} unsupported") from e - if use_quantized_matmul: - use_quantized_matmul = channel_size >= 32 and output_channel_size >= 32 - if use_quantized_matmul: - if dtype_dict[weights_dtype]["is_integer"]: - use_quantized_matmul = output_channel_size % 8 == 0 and channel_size % 8 == 0 - else: - use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 - - if group_size == 0: - if use_quantized_matmul and dtype_dict[weights_dtype]["num_bits"] >= 6: - group_size = -1 - elif is_linear_type: - group_size = 2 ** (2 + dtype_dict[weights_dtype]["num_bits"]) - else: - group_size = 2 ** (1 + dtype_dict[weights_dtype]["num_bits"]) - elif use_quantized_matmul and dtype_dict[weights_dtype]["num_bits"] == 8: - group_size = -1 # override user value, re-quantizing 8bit into 8bit is pointless - elif group_size != -1 and not is_linear_type: - group_size = max(group_size // 2, 1) - - if group_size > 0: - if group_size >= channel_size: - group_size = channel_size - num_of_groups = 1 - else: - num_of_groups = channel_size // group_size - while num_of_groups * group_size != channel_size: # find something divisible - num_of_groups -= 1 - if num_of_groups <= 1: - group_size = channel_size - num_of_groups = 1 - break - group_size = channel_size // num_of_groups - group_size = int(group_size) - num_of_groups = int(num_of_groups) - - if num_of_groups > 1: - result_shape = layer.weight.shape - new_shape = list(result_shape) - if is_conv_type: - # output_channel_size, channel_size, X, X - # output_channel_size, num_of_groups, group_size, X, X - new_shape[1] = group_size - new_shape.insert(1, num_of_groups) - reduction_axes = 2 - elif is_conv_transpose_type: - #channel_size, output_channel_size, X, X - #num_of_groups, group_size, output_channel_size, X, X - new_shape[0] = group_size - new_shape.insert(0, num_of_groups) - reduction_axes = 1 - elif is_linear_type: - # output_channel_size, channel_size - # output_channel_size, num_of_groups, group_size - last_dim_index = layer.weight.ndim - new_shape[last_dim_index - 1 : last_dim_index] = (num_of_groups, group_size) - layer.weight.data = layer.weight.reshape(new_shape) - - layer.weight.requires_grad = False - if return_device is None: - return_device = layer.weight.device - if quantization_device is not None: - layer.weight.data = layer.weight.to(quantization_device, non_blocking=non_blocking) - if layer.weight.dtype != torch.float32: - layer.weight.data = layer.weight.to(dtype=torch.float32) - - layer.weight.data, scale, zero_point = quantize_weight(layer.weight, reduction_axes, weights_dtype) - if not dequantize_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): - scale = scale.to(torch_dtype) - if zero_point is not None: - zero_point = zero_point.to(torch_dtype) - - re_quantize_for_matmul = (num_of_groups > 1 or zero_point is not None) - if use_quantized_matmul and not re_quantize_for_matmul: - if is_conv_type: - result_shape = layer.weight.shape - layer.weight.data = layer.weight.reshape(output_channel_size, -1) - scale.transpose_(0,1) - layer.weight.transpose_(0,1) - if not dtype_dict[weights_dtype]["is_integer"]: - weight_stride = layer.weight.stride() - if not (weight_stride[0] == 1 and weight_stride[1] > 1): - layer.weight.data = layer.weight.t().contiguous().t() - if not use_tensorwise_fp8_matmul: - scale = scale.to(torch.float32) - - layer.sdnq_dequantizer = dequantizer_dict[weights_dtype]( - scale=scale, - zero_point=zero_point, - quantized_weight_shape=layer.weight.shape, - result_dtype=torch_dtype, - result_shape=result_shape, - original_shape=original_shape, - weights_dtype=weights_dtype, - use_quantized_matmul=use_quantized_matmul, - re_quantize_for_matmul=re_quantize_for_matmul, - ) - layer.weight.data = layer.sdnq_dequantizer.pack_weight(layer.weight).to(return_device, non_blocking=non_blocking) - layer.sdnq_dequantizer = layer.sdnq_dequantizer.to(return_device, non_blocking=non_blocking) - - layer.forward = get_forward_func(layer_class_name, use_quantized_matmul, dtype_dict[weights_dtype]["is_integer"], use_tensorwise_fp8_matmul) - layer.forward = layer.forward.__get__(layer, layer.__class__) - return layer - - -def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, dequantize_fp32=False, non_blocking=False, quantization_device=None, return_device=None, modules_to_not_convert: List[str] = None, modules_dtype_dict: Dict[str, List[str]] = None, op=None): # pylint: disable=unused-argument - has_children = list(model.children()) - if not has_children: - return model - if modules_to_not_convert is None: - modules_to_not_convert = [] - if modules_dtype_dict is None: - modules_dtype_dict = {} - for param_name, module in model.named_children(): - if param_name in modules_to_not_convert: - continue - if hasattr(module, "weight") and module.weight is not None: - if len(modules_dtype_dict.keys()) > 0: - for key, value in modules_dtype_dict.items(): - if param_name in value: - key = key.lower() - if key in {"8bit", "8bits"}: - if dtype_dict[weights_dtype]["num_bits"] != 8: - weights_dtype = "int8" - elif key.startswith("minimum_"): - minimum_bits_str = key.removeprefix("minimum_").removesuffix("bits").removesuffix("bit") - minimum_bits = int(minimum_bits_str) - if dtype_dict[weights_dtype]["num_bits"] < minimum_bits: - weights_dtype = "int" + minimum_bits_str - if minimum_bits <= 4: - weights_dtype = "u" + weights_dtype - else: - weights_dtype = key - - module = sdnq_quantize_layer( - module, - weights_dtype=weights_dtype, - torch_dtype=torch_dtype, - group_size=group_size, - quant_conv=quant_conv, - use_quantized_matmul=use_quantized_matmul, - use_quantized_matmul_conv=use_quantized_matmul_conv, - dequantize_fp32=dequantize_fp32, - non_blocking=non_blocking, - quantization_device=quantization_device, - return_device=return_device, - param_name=param_name, - ) - module = apply_sdnq_to_module( - module, - weights_dtype=weights_dtype, - torch_dtype=torch_dtype, - group_size=group_size, - quant_conv=quant_conv, - use_quantized_matmul=use_quantized_matmul, - use_quantized_matmul_conv=use_quantized_matmul_conv, - dequantize_fp32=dequantize_fp32, - non_blocking=non_blocking, - quantization_device=quantization_device, - return_device=return_device, - modules_to_not_convert=modules_to_not_convert, - modules_dtype_dict=modules_dtype_dict, - op=op, - ) - return model - - -class SDNQQuantizer(DiffusersQuantizer): - r""" - Diffusers Quantizer for SDNQ - """ - - requires_parameters_quantization = True - use_keep_in_fp32_modules = True - requires_calibration = False - required_packages = None - torch_dtype = None - - def __init__(self, quantization_config, **kwargs): - super().__init__(quantization_config, **kwargs) - self.modules_to_not_convert = [] - - def check_if_quantized_param( - self, - model, - param_value: "torch.Tensor", - param_name: str, - state_dict: Dict[str, Any], # pylint: disable=unused-argument - **kwargs, # pylint: disable=unused-argument - ): - if param_name.endswith(".weight"): - split_param_name = param_name.split(".") - if ( - param_name not in self.modules_to_not_convert - and not any(param in split_param_name for param in self.modules_to_not_convert) - and not any("*" in param and re.match(param.replace(".*", "\\.*").replace("*", ".*"), param_name) for param in self.modules_to_not_convert) - ): - layer_class_name = get_module_from_name(model, param_name)[0].__class__.__name__ - if layer_class_name in allowed_types: - if layer_class_name in conv_types or layer_class_name in conv_transpose_types: - if self.quantization_config.quant_conv: - return True - else: - return True - with devices.inference_context(): - param_value.data = param_value.clone() # safetensors is unable to release the cpu memory without this - return False - - def check_quantized_param(self, *args, **kwargs) -> bool: - """ - needed for transformers compatibilty, returns self.check_if_quantized_param - """ - return self.check_if_quantized_param(*args, **kwargs) - - @devices.inference_context() - def create_quantized_param( # pylint: disable=arguments-differ - self, - model, - param_value: torch.FloatTensor, - param_name: str, - target_device: torch.device, - state_dict: Dict[str, Any], # pylint: disable=unused-argument - unexpected_keys: List[str], # pylint: disable=unused-argument - **kwargs, # pylint: disable=unused-argument - ): - weights_dtype = self.quantization_config.weights_dtype - if len(self.quantization_config.modules_dtype_dict.keys()) > 0: - split_param_name = param_name.split(".") - for key, value in self.quantization_config.modules_dtype_dict.items(): - if ( - param_name in value - or any(param in split_param_name for param in value) - or any("*" in param and re.match(param.replace(".*", "\\.*").replace("*", ".*"), param_name) for param in value) - ): - key = key.lower() - if key in {"8bit", "8bits"}: - if dtype_dict[weights_dtype]["num_bits"] != 8: - weights_dtype = "int8" - elif key.startswith("minimum_"): - minimum_bits_str = key.removeprefix("minimum_").removesuffix("bits").removesuffix("bit") - minimum_bits = int(minimum_bits_str) - if dtype_dict[weights_dtype]["num_bits"] < minimum_bits: - weights_dtype = "int" + minimum_bits_str - if minimum_bits <= 4: - weights_dtype = "u" + weights_dtype - else: - weights_dtype = key - - if self.quantization_config.return_device is not None: - return_device = self.quantization_config.return_device - else: - return_device = target_device - - if self.quantization_config.quantization_device is not None: - target_device = self.quantization_config.quantization_device - - if param_value.dtype == torch.float32 and devices.same_device(param_value.device, target_device): - param_value = param_value.clone() - else: - param_value = param_value.to(target_device, non_blocking=self.quantization_config.non_blocking).to(dtype=torch.float32) - - layer, _ = get_module_from_name(model, param_name) - layer.weight = torch.nn.Parameter(param_value, requires_grad=False) - layer = sdnq_quantize_layer( - layer, - weights_dtype=weights_dtype, - torch_dtype=self.torch_dtype, - group_size=self.quantization_config.group_size, - quant_conv=self.quantization_config.quant_conv, - use_quantized_matmul=self.quantization_config.use_quantized_matmul, - use_quantized_matmul_conv=self.quantization_config.use_quantized_matmul_conv, - dequantize_fp32=self.quantization_config.dequantize_fp32, - non_blocking=self.quantization_config.non_blocking, - quantization_device=None, - return_device=return_device, - param_name=param_name, - ) - - def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: - max_memory = {key: val * 0.80 for key, val in max_memory.items()} - return max_memory - - def adjust_target_dtype(self, target_dtype: torch.dtype) -> torch.dtype: # pylint: disable=unused-argument,arguments-renamed - return dtype_dict[self.quantization_config.weights_dtype]["target_dtype"] - - def update_torch_dtype(self, torch_dtype: torch.dtype = None) -> torch.dtype: - if torch_dtype is None: - torch_dtype = devices.dtype - self.torch_dtype = torch_dtype - return torch_dtype - - def _process_model_before_weight_loading( # pylint: disable=arguments-differ - self, - model, - device_map, # pylint: disable=unused-argument - keep_in_fp32_modules: List[str] = None, - **kwargs, # pylint: disable=unused-argument - ): - if keep_in_fp32_modules is not None: - self.modules_to_not_convert.extend(keep_in_fp32_modules) - elif getattr(model, "_keep_in_fp32_modules", None) is not None: - self.modules_to_not_convert.extend(model._keep_in_fp32_modules) # pylint: disable=protected-access - if getattr(model, "_skip_layerwise_casting_patterns", None) is not None: - self.modules_to_not_convert.extend(model._skip_layerwise_casting_patterns) # pylint: disable=protected-access - self.modules_to_not_convert.extend(self.quantization_config.modules_to_not_convert) - self.quantization_config.modules_to_not_convert = self.modules_to_not_convert - model.config.quantization_config = self.quantization_config - - def _process_model_after_weight_loading(self, model, **kwargs): # pylint: disable=unused-argument - if shared.opts.diffusers_offload_mode != "none": - model = model.to(devices.cpu) - devices.torch_gc(force=True, reason='sdnq') - return model - - def get_accelerator_warm_up_factor(self): - return 32 // dtype_dict[self.quantization_config.weights_dtype]["num_bits"] - - def get_cuda_warm_up_factor(self): - """ - needed for transformers compatibilty, returns self.get_accelerator_warm_up_factor - """ - return self.get_accelerator_warm_up_factor() - - def update_tp_plan(self, config): - """ - needed for transformers compatibilty, no-op function - """ - return config - - def update_unexpected_keys(self, model, unexpected_keys: List[str], prefix: str) -> List[str]: # pylint: disable=unused-argument - """ - needed for transformers compatibilty, no-op function - """ - return unexpected_keys - - def update_missing_keys_after_loading(self, model, missing_keys: List[str], prefix: str) -> List[str]: # pylint: disable=unused-argument - """ - needed for transformers compatibilty, no-op function - """ - return missing_keys - - def update_expected_keys(self, model, expected_keys: List[str], loaded_keys: List[str]) -> List[str]: # pylint: disable=unused-argument - """ - needed for transformers compatibilty, no-op function - """ - return expected_keys - - def update_param_name(self, param_name: str) -> str: - """ - needed for transformers compatibilty, no-op function - """ - return param_name - - def update_dtype(self, dtype: torch.dtype) -> torch.dtype: - """ - needed for transformers compatibilty, no-op function - """ - return dtype - - @property - def is_trainable(self): - return False - - @property - def is_serializable(self): - return True - - @property - def is_compileable(self): - return True - - -@dataclass -class SDNQConfig(QuantizationConfigMixin): - """ - This is a wrapper class about all possible attributes and features that you can play with a model that has been - loaded using `sdnq`. - - Args: - weights_dtype (`str`, *optional*, defaults to `"int8"`): - The target dtype for the weights after quantization. Supported values are: - ("int8", "int7", "int6", "int5", "int4", "int3", "int2", "uint8", "uint7", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz") - weights_dtype (`int`, *optional*, defaults to `0`): - Used to decide how many elements of a tensor will share the same quantization group. - quant_conv (`bool`, *optional*, defaults to `False`): - Enabling this option will quantize the convolutional layers in UNet models too. - use_quantized_matmul (`bool`, *optional*, defaults to `False`): - Enabling this option will use quantized INT8 or FP8 MatMul instead of BF16 / FP16. - use_quantized_matmul_conv (`bool`, *optional*, defaults to `False`): - Same as use_quantized_matmul_conv but for the convolutional layers with UNets like SDXL. - dequantize_fp32 (`bool`, *optional*, defaults to `False`): - Enabling this option will use FP32 on the dequantization step. - non_blocking (`bool`, *optional*, defaults to `False`): - Enabling this option will use non blocking ops when moving layers between the quantization device and the return device. - quantization_device (`torch.device`, *optional*, defaults to `None`): - Used to set which device will be used for the quantization calculation on model load. - return_device (`torch.device`, *optional*, defaults to `None`): - Used to set which device will the quantized weights be sent back to. - modules_to_not_convert (`list`, *optional*, default to `None`): - The list of modules to not quantize, useful for quantizing models that explicitly require to have some - modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers). - modules_dtype_dict (`dict`, *optional*, default to `None`): - The dict of dtypes and list of modules, useful for quantizing some modules with a different dtype. - """ - - def __init__( # pylint: disable=super-init-not-called - self, - weights_dtype: str = "int8", - group_size: int = 0, - quant_conv: bool = False, - use_quantized_matmul: bool = False, - use_quantized_matmul_conv: bool = False, - dequantize_fp32: bool = False, - non_blocking: bool = False, - quantization_device: Optional[torch.device] = None, - return_device: Optional[torch.device] = None, - modules_to_not_convert: Optional[List[str]] = None, - modules_dtype_dict: Optional[Dict[str, List[str]]] = None, - **kwargs, # pylint: disable=unused-argument - ): - self.weights_dtype = weights_dtype - self.quant_method = QuantizationMethod.SDNQ - self.group_size = group_size - self.quant_conv = quant_conv - self.use_quantized_matmul = use_quantized_matmul - self.use_quantized_matmul_conv = use_quantized_matmul_conv - self.dequantize_fp32 = dequantize_fp32 - self.non_blocking = non_blocking - self.quantization_device = quantization_device - self.return_device = return_device - self.modules_to_not_convert = modules_to_not_convert - self.modules_dtype_dict = modules_dtype_dict - self.post_init() - self.is_integer = dtype_dict[self.weights_dtype]["is_integer"] - - def post_init(self): - r""" - Safety checker that arguments are correct - """ - accepted_weights = ["int8", "int7", "int6", "int5", "int4", "int3", "int2", "uint8", "uint7", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz"] - if self.weights_dtype not in accepted_weights: - raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}") - - if self.modules_to_not_convert is None: - self.modules_to_not_convert = [] - elif not isinstance(self.modules_to_not_convert, list): - self.modules_to_not_convert = [self.modules_to_not_convert] - - if self.modules_dtype_dict is None: - self.modules_dtype_dict = {} - - -import diffusers.quantizers.auto # noqa: E402,RUF100 # pylint: disable=wrong-import-order -diffusers.quantizers.auto.AUTO_QUANTIZER_MAPPING["sdnq"] = SDNQQuantizer -diffusers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig - -import transformers.quantizers.auto # noqa: E402,RUF100 # pylint: disable=wrong-import-order -transformers.quantizers.auto.AUTO_QUANTIZER_MAPPING["sdnq"] = SDNQQuantizer -transformers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig +__all__ = [ + "QuantizationMethod", + "SDNQConfig", + "SDNQQuantizer", + "apply_sdnq_to_module", + "load_sdnq_model", + "save_sdnq_model", + "sdnq_post_load_quant", + "sdnq_quantize_layer", +] diff --git a/modules/sdnq/common.py b/modules/sdnq/common.py index fab48c4b9..c5ff8ab9a 100644 --- a/modules/sdnq/common.py +++ b/modules/sdnq/common.py @@ -2,9 +2,9 @@ import os import torch -from modules import shared -torch_version = float(torch.__version__[:3]) +from modules import shared, devices + dtype_dict = { "int8": {"min": -128, "max": 127, "num_bits": 8, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.int8, "is_unsigned": False, "is_integer": True}, @@ -31,14 +31,74 @@ if hasattr(torch, "float8_e4m3fnuz"): if hasattr(torch, "float8_e5m2fnuz"): dtype_dict["float8_e5m2fnuz"] = {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": "fp8", "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False} -use_torch_compile = shared.opts.sdnq_dequantize_compile # this setting requires a full restart of the webui to apply -use_tensorwise_fp8_matmul = os.environ.get('SDNQ_USE_TENSORWISE_FP8_MATMUL', "1").lower() not in {"0", "false", "no"} # row-wise FP8 only exist on H100 hardware, sdnq will use software row-wise with tensorwise hardware with this setting +linear_types = {"Linear"} +conv_types = {"Conv1d", "Conv2d", "Conv3d"} +conv_transpose_types = {"ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d"} +allowed_types = set.union(linear_types, conv_types, conv_transpose_types) +accepted_weights = set(dtype_dict.keys()) + +use_torch_compile = shared.opts.sdnq_dequantize_compile # this setting requires a full restart of the webui to apply +is_rdna2 = bool(devices.backend == "rocm" and int(getattr(torch.cuda.get_device_properties(devices.device), "gcnArchName", "gfx0000")[3:]) < 1100) + + +if os.environ.get("SDNQ_USE_TENSORWISE_FP8_MM", None) is None: + # row-wise FP8 only exist on H100 hardware, sdnq will use software row-wise with tensorwise hardware with this setting + use_tensorwise_fp8_matmul = bool(devices.backend == "cuda" and torch.cuda.get_device_capability(devices.device) < (9,0)) +else: + use_tensorwise_fp8_matmul = os.environ.get("SDNQ_USE_TENSORWISE_FP8_MM", "0").lower() not in {"0", "false", "no"} + +if os.environ.get("SDNQ_USE_CONTIGUOUS_MM", None) is None: + use_contiguous_mm = bool(is_rdna2 or devices.backend in {"cpu", "ipex", "zluda"}) +else: + use_contiguous_mm = bool(os.environ.get("SDNQ_USE_CONTIGUOUS_MM", "0").lower() not in {"0", "false", "no"}) + +if os.environ.get("SDNQ_USE_TRITON_MM", None) is None: + use_triton_mm = bool(is_rdna2 or devices.backend == "zluda") +else: + use_triton_mm = bool(os.environ.get("SDNQ_USE_TRITON_MM", "0").lower() not in {"0", "false", "no"}) + + +if use_triton_mm: + try: + from .triton_mm import int_mm + int_mm_func = int_mm + except ImportError: + int_mm_func = torch._int_mm +else: + int_mm_func = torch._int_mm -linear_types = ("Linear",) -conv_types = ("Conv1d", "Conv2d", "Conv3d") -conv_transpose_types = ("ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d") -allowed_types = linear_types + conv_types + conv_transpose_types if use_torch_compile: torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) torch._dynamo.config.accumulated_recompile_limit = max(8192, torch._dynamo.config.accumulated_recompile_limit) + def compile_func(fn, **kwargs): + if kwargs.get("fullgraph", None) is None: + kwargs["fullgraph"] = True + if kwargs.get("dynamic", None) is None: + kwargs["dynamic"] = False + return torch.compile(fn, **kwargs) +else: + def compile_func(fn, **kwargs): # pylint: disable=unused-argument + return fn + + +module_skip_keys_dict = { + "FluxTransformer2DModel": [ + ["single_transformer_blocks.0.norm.linear.weight", ".time_text_embed", ".context_embedder", ".x_embedder", ".proj_out", ".norm_out", "pos_embed"], + {} + ], + "ChromaTransformer2DModel": [ + ["distilled_guidance_layer", ".time_text_embed", ".context_embedder", ".x_embedder", ".proj_out", ".norm_out", "pos_embed"], + {} + ], + "QwenImageTransformer2DModel": [ + ["transformer_blocks.0.img_mod.1.weight", ".time_text_embed", ".txt_in", ".img_in", ".proj_out", ".norm_out", "pos_embed"], + {} + ], + "NaDiT": [ + [".emb_in", ".txt_in", ".vid_in", ".emb_scale", ".vid_out", ".vid_out_norm", ".vid_out_ada"], + {} + ], +} + +module_skip_keys_dict["NaDiTUpscaler"] = module_skip_keys_dict["NaDiT"] diff --git a/modules/sdnq/dequantizer.py b/modules/sdnq/dequantizer.py index 9ac902c57..2b9998adb 100644 --- a/modules/sdnq/dequantizer.py +++ b/modules/sdnq/dequantizer.py @@ -1,26 +1,44 @@ # pylint: disable=redefined-builtin,no-member,protected-access -from typing import Tuple +from typing import Tuple, Optional import torch -from .common import dtype_dict, use_torch_compile +from .common import dtype_dict, compile_func, use_contiguous_mm, use_tensorwise_fp8_matmul from .packed_int import pack_int_symetric, unpack_int_symetric, pack_int_asymetric, unpack_int_asymetric -def dequantize_asymmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, dtype: torch.dtype, result_shape: torch.Size) -> torch.FloatTensor: - result = torch.addcmul(zero_point, weight.to(dtype=scale.dtype), scale).to(dtype=dtype) +def dequantize_asymmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, dtype: torch.dtype, result_shape: torch.Size, svd_up: Optional[torch.FloatTensor] = None, svd_down: Optional[torch.FloatTensor] = None, skip_quantized_matmul: bool = False) -> torch.FloatTensor: + result = torch.addcmul(zero_point, weight.to(dtype=scale.dtype), scale) if result_shape is not None: result = result.view(result_shape) + if svd_up is not None: + if skip_quantized_matmul: + svd_up, svd_down = svd_up.t(), svd_down.t() + if result.ndim > 2 and weight.ndim > 2: # convs + result = result.add_(torch.mm(svd_up, svd_down).unflatten(-1, (*result.shape[1:],))) + else: + result = result.addmm_(svd_up, svd_down) + if dtype is not None: + result = result.to(dtype=dtype) return result -def dequantize_symmetric(weight: torch.CharTensor, scale: torch.FloatTensor, dtype: torch.dtype, result_shape: torch.Size, skip_quantized_matmul: bool = False) -> torch.FloatTensor: - result = weight.to(dtype=scale.dtype).mul_(scale).to(dtype=dtype) +def dequantize_symmetric(weight: torch.CharTensor, scale: torch.FloatTensor, dtype: torch.dtype, result_shape: torch.Size, svd_up: Optional[torch.FloatTensor] = None, svd_down: Optional[torch.FloatTensor] = None, skip_quantized_matmul: bool = False) -> torch.FloatTensor: + result = weight.to(dtype=scale.dtype).mul_(scale) if skip_quantized_matmul: result.t_() if result_shape is not None: result = result.view(result_shape) + if svd_up is not None: + if skip_quantized_matmul: + svd_up, svd_down = svd_up.t(), svd_down.t() + if result.ndim > 2 and weight.ndim > 2: # convs + result = result.add_(torch.mm(svd_up, svd_down).unflatten(-1, (*result.shape[1:],))) + else: + result = result.addmm_(svd_up, svd_down) + if dtype is not None: + result = result.to(dtype=dtype) return result @@ -28,12 +46,12 @@ def dequantize_symmetric_with_bias(weight: torch.CharTensor, scale: torch.FloatT return torch.addcmul(bias, weight.to(dtype=scale.dtype), scale).to(dtype=dtype).view(result_shape) -def dequantize_packed_int_asymmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str) -> torch.FloatTensor: - return dequantize_asymmetric(unpack_int_asymetric(weight, shape, weights_dtype), scale, zero_point, dtype, result_shape) +def dequantize_packed_int_asymmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str, svd_up: Optional[torch.FloatTensor] = None, svd_down: Optional[torch.FloatTensor] = None, skip_quantized_matmul: bool = False) -> torch.FloatTensor: + return dequantize_asymmetric(unpack_int_asymetric(weight, shape, weights_dtype), scale, zero_point, dtype, result_shape, svd_up=svd_up, svd_down=svd_down, skip_quantized_matmul=skip_quantized_matmul) -def dequantize_packed_int_symmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str, skip_quantized_matmul: bool = False) -> torch.FloatTensor: - return dequantize_symmetric(unpack_int_symetric(weight, shape, weights_dtype, dtype=scale.dtype), scale, dtype, result_shape, skip_quantized_matmul=skip_quantized_matmul) +def dequantize_packed_int_symmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str, svd_up: Optional[torch.FloatTensor] = None, svd_down: Optional[torch.FloatTensor] = None, skip_quantized_matmul: bool = False) -> torch.FloatTensor: + return dequantize_symmetric(unpack_int_symetric(weight, shape, weights_dtype, dtype=scale.dtype), scale, dtype, result_shape, svd_up=svd_up, svd_down=svd_down, skip_quantized_matmul=skip_quantized_matmul) def quantize_int8(input: torch.FloatTensor, dim: int = -1) -> Tuple[torch.CharTensor, torch.FloatTensor]: @@ -42,39 +60,69 @@ def quantize_int8(input: torch.FloatTensor, dim: int = -1) -> Tuple[torch.CharTe return input, scale -def quantize_fp8(input: torch.FloatTensor, dim: int = -1) -> Tuple[torch.Tensor, torch.FloatTensor]: - scale = torch.amax(input.abs(), dim=dim, keepdims=True).div_(448) - input = torch.div(input, scale).nan_to_num_().clamp_(-448, 448).to(dtype=torch.float8_e4m3fn) +def quantize_fp8(input: torch.FloatTensor, dim: int = -1, is_e5: bool = False) -> Tuple[torch.Tensor, torch.FloatTensor]: + max_range = 57344 if is_e5 else 448 + fp8_dtype = torch.float8_e5m2 if is_e5 else torch.float8_e4m3fn + scale = torch.amax(input.abs(), dim=dim, keepdims=True).div_(max_range) + input = torch.div(input, scale).nan_to_num_().clamp_(-max_range, max_range).to(dtype=fp8_dtype) return input, scale -def re_quantize_matmul_asymmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, result_shape: torch.Size) -> Tuple[torch.CharTensor, torch.FloatTensor]: - result = dequantize_asymmetric(weight, scale, zero_point, scale.dtype, result_shape) - if result.ndim > 2: # convs - result = result.flatten(1,-1) - return quantize_int8(result.t_(), dim=0) +def re_quantize_int8(weight: torch.FloatTensor) -> Tuple[torch.CharTensor, torch.FloatTensor]: + if weight.ndim > 2: # convs + weight = weight.flatten(1,-1) + weight = weight.t() + if use_contiguous_mm: + weight = weight.contiguous() + weight, scale = quantize_int8(weight, dim=0) + return weight, scale -def re_quantize_matmul_symmetric(weight: torch.CharTensor, scale: torch.FloatTensor, result_shape: torch.Size) -> Tuple[torch.CharTensor, torch.FloatTensor]: - result = dequantize_symmetric(weight, scale, scale.dtype, result_shape) - if result.ndim > 2: # convs - result = result.flatten(1,-1) - return quantize_int8(result.t_(), dim=0) +def re_quantize_fp8(weight: torch.FloatTensor, is_e5: bool = False) -> Tuple[torch.CharTensor, torch.FloatTensor]: + if weight.ndim > 2: # convs + weight = weight.flatten(1,-1) + weight, scale = quantize_fp8(weight.t(), dim=0, is_e5=is_e5) + if not use_tensorwise_fp8_matmul: + scale = scale.to(dtype=torch.float32) + return weight, scale -def re_quantize_matmul_packed_int_asymmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, shape: torch.Size, result_shape: torch.Size, weights_dtype: str) -> torch.FloatTensor: - return re_quantize_matmul_asymmetric(unpack_int_asymetric(weight, shape, weights_dtype), scale, zero_point, result_shape) +def re_quantize_matmul_asymmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, result_shape: torch.Size, svd_up: Optional[torch.FloatTensor] = None, svd_down: Optional[torch.FloatTensor] = None) -> Tuple[torch.CharTensor, torch.FloatTensor]: + return re_quantize_int8(dequantize_asymmetric(weight, scale, zero_point, scale.dtype, result_shape, svd_up=svd_up, svd_down=svd_down)) -def re_quantize_matmul_packed_int_symmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, shape: torch.Size, result_shape: torch.Size, weights_dtype: str) -> torch.FloatTensor: - return re_quantize_matmul_symmetric(unpack_int_symetric(weight, shape, weights_dtype, dtype=scale.dtype), scale, result_shape) +def re_quantize_matmul_symmetric(weight: torch.CharTensor, scale: torch.FloatTensor, result_shape: torch.Size, svd_up: Optional[torch.FloatTensor] = None, svd_down: Optional[torch.FloatTensor] = None) -> Tuple[torch.CharTensor, torch.FloatTensor]: + return re_quantize_int8(dequantize_symmetric(weight, scale, scale.dtype, result_shape, svd_up=svd_up, svd_down=svd_down)) + + +def re_quantize_matmul_packed_int_asymmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, shape: torch.Size, result_shape: torch.Size, weights_dtype: str, svd_up: Optional[torch.FloatTensor] = None, svd_down: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + return re_quantize_matmul_asymmetric(unpack_int_asymetric(weight, shape, weights_dtype), scale, zero_point, result_shape, svd_up=svd_up, svd_down=svd_down) + + +def re_quantize_matmul_packed_int_symmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, shape: torch.Size, result_shape: torch.Size, weights_dtype: str, svd_up: Optional[torch.FloatTensor] = None, svd_down: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + return re_quantize_matmul_symmetric(unpack_int_symetric(weight, shape, weights_dtype, dtype=scale.dtype), scale, result_shape, svd_up=svd_up, svd_down=svd_down) + + +def dequantize_sdnq_model(model): + if hasattr(model, "sdnq_dequantizer"): + model.weight = torch.nn.Parameter(model.sdnq_dequantizer(model.weight, model.scale, model.zero_point, model.svd_up, model.svd_down)) + del model.sdnq_dequantizer, model.scale, model.zero_point, model.svd_up, model.svd_down + return model + has_children = list(model.children()) + if not has_children: + return model + for module in model.children(): + if hasattr(module, "sdnq_dequantizer"): + module.weight = torch.nn.Parameter(module.sdnq_dequantizer(module.weight, module.scale, module.zero_point, module.svd_up, module.svd_down)) + del module.sdnq_dequantizer, module.scale, module.zero_point, module.svd_up, module.svd_down + else: + module = dequantize_sdnq_model(module) + return model class AsymmetricWeightsDequantizer(torch.nn.Module): def __init__( self, - scale: torch.FloatTensor, - zero_point: torch.FloatTensor, result_dtype: torch.dtype, result_shape: torch.Size, original_shape: torch.Size, @@ -89,23 +137,20 @@ class AsymmetricWeightsDequantizer(torch.nn.Module): self.re_quantize_for_matmul = True self.result_dtype = result_dtype self.result_shape = result_shape - self.register_buffer("scale", scale) - self.register_buffer("zero_point", zero_point) def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) - def re_quantize_matmul(self, weight, **kwargs): # pylint: disable=unused-argument - return re_quantize_matmul_asymmetric_compiled(weight, self.scale, self.zero_point, self.result_shape) + def re_quantize_matmul(self, weight, scale, zero_point, svd_up, svd_down, **kwargs): # pylint: disable=unused-argument + return re_quantize_matmul_asymmetric_compiled(weight, scale, zero_point, self.result_shape, svd_up=svd_up, svd_down=svd_down) - def forward(self, weight, **kwargs): # pylint: disable=unused-argument - return dequantize_asymmetric_compiled(weight, self.scale, self.zero_point, self.result_dtype, self.result_shape) + def forward(self, weight, scale, zero_point, svd_up, svd_down, skip_quantized_matmul=False): # pylint: disable=unused-argument + return dequantize_asymmetric_compiled(weight, scale, zero_point, self.result_dtype, self.result_shape, svd_up=svd_up, svd_down=svd_down, skip_quantized_matmul=skip_quantized_matmul) class SymmetricWeightsDequantizer(torch.nn.Module): def __init__( self, - scale: torch.FloatTensor, result_dtype: torch.dtype, result_shape: torch.Size, original_shape: torch.Size, @@ -121,24 +166,21 @@ class SymmetricWeightsDequantizer(torch.nn.Module): self.re_quantize_for_matmul = re_quantize_for_matmul self.result_dtype = result_dtype self.result_shape = result_shape - self.register_buffer("scale", scale) def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) - def re_quantize_matmul(self, weight, **kwargs): # pylint: disable=unused-argument - return re_quantize_matmul_symmetric_compiled(weight, self.scale, self.result_shape) + def re_quantize_matmul(self, weight, scale, zero_point, svd_up, svd_down, **kwargs): # pylint: disable=unused-argument + return re_quantize_matmul_symmetric_compiled(weight, scale, self.result_shape, svd_up=svd_up, svd_down=svd_down) - def forward(self, weight, skip_quantized_matmul=False, **kwargs): # pylint: disable=unused-argument + def forward(self, weight, scale, zero_point, svd_up, svd_down, skip_quantized_matmul=False): # pylint: disable=unused-argument skip_quantized_matmul = skip_quantized_matmul and not self.re_quantize_for_matmul - return dequantize_symmetric_compiled(weight, self.scale, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) + return dequantize_symmetric_compiled(weight, scale, self.result_dtype, self.result_shape, svd_up=svd_up, svd_down=svd_down, skip_quantized_matmul=skip_quantized_matmul) class PackedINTAsymmetricWeightsDequantizer(torch.nn.Module): def __init__( self, - scale: torch.FloatTensor, - zero_point: torch.FloatTensor, quantized_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, @@ -155,23 +197,20 @@ class PackedINTAsymmetricWeightsDequantizer(torch.nn.Module): self.quantized_weight_shape = quantized_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape - self.register_buffer("scale", scale) - self.register_buffer("zero_point", zero_point) def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: return pack_int_asymetric(weight, self.weights_dtype) - def re_quantize_matmul(self, weight, **kwargs): # pylint: disable=unused-argument - return re_quantize_matmul_packed_int_asymmetric_compiled(weight, self.scale, self.zero_point, self.quantized_weight_shape, self.result_shape, self.weights_dtype) + def re_quantize_matmul(self, weight, scale, zero_point, svd_up, svd_down, **kwargs): # pylint: disable=unused-argument + return re_quantize_matmul_packed_int_asymmetric_compiled(weight, scale, zero_point, self.quantized_weight_shape, self.result_shape, self.weights_dtype, svd_up=svd_up, svd_down=svd_down) - def forward(self, weight, **kwargs): # pylint: disable=unused-argument - return dequantize_packed_int_asymmetric_compiled(weight, self.scale, self.zero_point, self.quantized_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype) + def forward(self, weight, scale, zero_point, svd_up, svd_down, skip_quantized_matmul=False): # pylint: disable=unused-argument + return dequantize_packed_int_asymmetric_compiled(weight, scale, zero_point, self.quantized_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype, svd_up=svd_up, svd_down=svd_down, skip_quantized_matmul=skip_quantized_matmul) class PackedINTSymmetricWeightsDequantizer(torch.nn.Module): def __init__( self, - scale: torch.FloatTensor, quantized_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, @@ -189,17 +228,16 @@ class PackedINTSymmetricWeightsDequantizer(torch.nn.Module): self.quantized_weight_shape = quantized_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape - self.register_buffer("scale", scale) def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: return pack_int_symetric(weight, self.weights_dtype) - def re_quantize_matmul(self, weight, **kwargs): # pylint: disable=unused-argument - return re_quantize_matmul_packed_int_symmetric_compiled(weight, self.scale, self.quantized_weight_shape, self.result_shape, self.weights_dtype) + def re_quantize_matmul(self, weight, scale, zero_point, svd_up, svd_down, **kwargs): # pylint: disable=unused-argument + return re_quantize_matmul_packed_int_symmetric_compiled(weight, scale, self.quantized_weight_shape, self.result_shape, self.weights_dtype, svd_up=svd_up, svd_down=svd_down) - def forward(self, weight, skip_quantized_matmul=False, **kwargs): # pylint: disable=unused-argument + def forward(self, weight, scale, zero_point, svd_up, svd_down, skip_quantized_matmul=False): # pylint: disable=unused-argument skip_quantized_matmul = skip_quantized_matmul and not self.re_quantize_for_matmul - return dequantize_packed_int_symmetric_compiled(weight, self.scale, self.quantized_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype, skip_quantized_matmul=skip_quantized_matmul) + return dequantize_packed_int_symmetric_compiled(weight, scale, self.quantized_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype, svd_up=svd_up, svd_down=svd_down, skip_quantized_matmul=skip_quantized_matmul) dequantizer_dict = { @@ -226,21 +264,11 @@ dequantizer_dict = { } -if use_torch_compile: - dequantize_asymmetric_compiled = torch.compile(dequantize_asymmetric, fullgraph=True, dynamic=False) - dequantize_symmetric_compiled = torch.compile(dequantize_symmetric, fullgraph=True, dynamic=False) - dequantize_packed_int_asymmetric_compiled = torch.compile(dequantize_packed_int_asymmetric, fullgraph=True, dynamic=False) - dequantize_packed_int_symmetric_compiled = torch.compile(dequantize_packed_int_symmetric, fullgraph=True, dynamic=False) - re_quantize_matmul_asymmetric_compiled = torch.compile(re_quantize_matmul_asymmetric, fullgraph=True, dynamic=False) - re_quantize_matmul_symmetric_compiled = torch.compile(re_quantize_matmul_symmetric, fullgraph=True, dynamic=False) - re_quantize_matmul_packed_int_asymmetric_compiled = torch.compile(re_quantize_matmul_packed_int_asymmetric, fullgraph=True, dynamic=False) - re_quantize_matmul_packed_int_symmetric_compiled = torch.compile(re_quantize_matmul_packed_int_symmetric, fullgraph=True, dynamic=False) -else: - dequantize_asymmetric_compiled = dequantize_asymmetric - dequantize_symmetric_compiled = dequantize_symmetric - dequantize_packed_int_asymmetric_compiled = dequantize_packed_int_asymmetric - dequantize_packed_int_symmetric_compiled = dequantize_packed_int_symmetric - re_quantize_matmul_asymmetric_compiled = re_quantize_matmul_asymmetric - re_quantize_matmul_symmetric_compiled = re_quantize_matmul_symmetric - re_quantize_matmul_packed_int_asymmetric_compiled = re_quantize_matmul_packed_int_asymmetric - re_quantize_matmul_packed_int_symmetric_compiled = re_quantize_matmul_packed_int_symmetric +dequantize_asymmetric_compiled = compile_func(dequantize_asymmetric) +dequantize_symmetric_compiled = compile_func(dequantize_symmetric) +dequantize_packed_int_asymmetric_compiled = compile_func(dequantize_packed_int_asymmetric) +dequantize_packed_int_symmetric_compiled = compile_func(dequantize_packed_int_symmetric) +re_quantize_matmul_asymmetric_compiled = compile_func(re_quantize_matmul_asymmetric) +re_quantize_matmul_symmetric_compiled = compile_func(re_quantize_matmul_symmetric) +re_quantize_matmul_packed_int_asymmetric_compiled = compile_func(re_quantize_matmul_packed_int_asymmetric) +re_quantize_matmul_packed_int_symmetric_compiled = compile_func(re_quantize_matmul_packed_int_symmetric) diff --git a/modules/sdnq/layers/conv/conv_fp8.py b/modules/sdnq/layers/conv/conv_fp8.py index cfce8994a..bdaf27062 100644 --- a/modules/sdnq/layers/conv/conv_fp8.py +++ b/modules/sdnq/layers/conv/conv_fp8.py @@ -4,8 +4,9 @@ from typing import List import torch -from ...common import use_torch_compile # noqa: TID252 +from ...common import compile_func # noqa: TID252 from ..linear.linear_fp8 import quantize_fp8_matmul_input # noqa: TID252 +from ..linear.forward import check_mats # noqa: TID252 from .forward import get_conv_args, process_conv_input @@ -14,6 +15,8 @@ def conv_fp8_matmul( weight: torch.Tensor, bias: torch.FloatTensor, scale: torch.FloatTensor, + svd_up: torch.FloatTensor, + svd_down: torch.FloatTensor, result_shape: torch.Size, reversed_padding_repeated_twice: List[int], padding_mode: str, conv_type: int, @@ -22,12 +25,17 @@ def conv_fp8_matmul( ) -> torch.FloatTensor: return_dtype = input.dtype input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) + if svd_up is not None: + input = input.flatten(0,-2) + svd_bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) + input, input_scale = quantize_fp8_matmul_input(input) + input, weight = check_mats(input, weight) if groups == 1: if bias is not None and bias.dtype != torch.bfloat16: bias = bias.to(dtype=torch.bfloat16) - result = torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=torch.bfloat16).view(mm_output_shape).to(return_dtype) + result = torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=torch.bfloat16) else: scale = scale.view(groups, 1, scale.shape[1] // groups) input_scale = input_scale.view(groups, input_scale.shape[0] // groups, 1) @@ -43,7 +51,10 @@ def conv_fp8_matmul( else: for i in range(groups): result.append(torch._scaled_mm(input[:, i], weight[:, i], scale_a=input_scale[i], scale_b=scale[i], bias=None, out_dtype=torch.bfloat16)) - result = torch.cat(result, dim=-1).view(mm_output_shape).to(return_dtype) + result = torch.cat(result, dim=-1) + if svd_up is not None: + result.add_(svd_bias) + result = result.view(mm_output_shape).to(return_dtype) if conv_type == 1: result = result.transpose_(1,2) @@ -56,11 +67,11 @@ def conv_fp8_matmul( def quantized_conv_forward_fp8_matmul(self, input) -> torch.FloatTensor: if torch.numel(input) / input.shape[2] < 32: - return self._conv_forward(input, self.sdnq_dequantizer(self.weight, skip_quantized_matmul=True), self.bias) + return self._conv_forward(input, self.sdnq_dequantizer(self.weight, self.scale, self.zero_point, self.svd_up, self.svd_down, skip_quantized_matmul=True), self.bias) conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) return conv_fp8_matmul( input, self.weight, self.bias, - self.sdnq_dequantizer.scale, + self.scale, self.svd_up, self.svd_down, self.sdnq_dequantizer.result_shape, self._reversed_padding_repeated_twice, self.padding_mode, conv_type, @@ -68,5 +79,4 @@ def quantized_conv_forward_fp8_matmul(self, input) -> torch.FloatTensor: ) -if use_torch_compile: - conv_fp8_matmul = torch.compile(conv_fp8_matmul, fullgraph=True, dynamic=False) +conv_fp8_matmul = compile_func(conv_fp8_matmul) diff --git a/modules/sdnq/layers/conv/conv_fp8_tensorwise.py b/modules/sdnq/layers/conv/conv_fp8_tensorwise.py index a21699855..2dc9fbda3 100644 --- a/modules/sdnq/layers/conv/conv_fp8_tensorwise.py +++ b/modules/sdnq/layers/conv/conv_fp8_tensorwise.py @@ -4,9 +4,10 @@ from typing import List import torch -from ...common import use_torch_compile # noqa: TID252 +from ...common import compile_func # noqa: TID252 from ...dequantizer import dequantize_symmetric, dequantize_symmetric_with_bias # noqa: TID252 from ..linear.linear_fp8_tensorwise import quantize_fp8_matmul_input_tensorwise # noqa: TID252 +from ..linear.forward import check_mats # noqa: TID252 from .forward import get_conv_args, process_conv_input @@ -15,6 +16,8 @@ def conv_fp8_matmul_tensorwise( weight: torch.Tensor, bias: torch.FloatTensor, scale: torch.FloatTensor, + svd_up: torch.FloatTensor, + svd_down: torch.FloatTensor, result_shape: torch.Size, reversed_padding_repeated_twice: List[int], padding_mode: str, conv_type: int, @@ -23,7 +26,15 @@ def conv_fp8_matmul_tensorwise( ) -> torch.FloatTensor: return_dtype = input.dtype input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) + if svd_up is not None: + input = input.flatten(0,-2) + if bias is not None: + bias = torch.addmm(bias, torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) + else: + bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) + input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) + input, weight = check_mats(input, weight) dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) if groups == 1: @@ -51,11 +62,11 @@ def conv_fp8_matmul_tensorwise( def quantized_conv_forward_fp8_matmul_tensorwise(self, input) -> torch.FloatTensor: if torch.numel(input) / input.shape[2] < 32: - return self._conv_forward(input, self.sdnq_dequantizer(self.weight, skip_quantized_matmul=True), self.bias) + return self._conv_forward(input, self.sdnq_dequantizer(self.weight, self.scale, self.zero_point, self.svd_up, self.svd_down, skip_quantized_matmul=True), self.bias) conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) return conv_fp8_matmul_tensorwise( input, self.weight, self.bias, - self.sdnq_dequantizer.scale, + self.scale, self.svd_up, self.svd_down, self.sdnq_dequantizer.result_shape, self._reversed_padding_repeated_twice, self.padding_mode, conv_type, @@ -63,5 +74,4 @@ def quantized_conv_forward_fp8_matmul_tensorwise(self, input) -> torch.FloatTens ) -if use_torch_compile: - conv_fp8_matmul_tensorwise = torch.compile(conv_fp8_matmul_tensorwise, fullgraph=True, dynamic=False) +conv_fp8_matmul_tensorwise = compile_func(conv_fp8_matmul_tensorwise) diff --git a/modules/sdnq/layers/conv/conv_int8.py b/modules/sdnq/layers/conv/conv_int8.py index cd18b1db6..3332537d1 100644 --- a/modules/sdnq/layers/conv/conv_int8.py +++ b/modules/sdnq/layers/conv/conv_int8.py @@ -4,10 +4,11 @@ from typing import List import torch -from ...common import use_torch_compile # noqa: TID252 +from ...common import compile_func, int_mm_func # noqa: TID252 from ...packed_int import unpack_int_symetric # noqa: TID252 from ...dequantizer import dequantize_symmetric, dequantize_symmetric_with_bias # noqa: TID252 from ..linear.linear_int8 import quantize_int8_matmul_input # noqa: TID252 +from ..linear.forward import check_mats # noqa: TID252 from .forward import get_conv_args, process_conv_input @@ -16,6 +17,8 @@ def conv_int8_matmul( weight: torch.CharTensor, bias: torch.FloatTensor, scale: torch.FloatTensor, + svd_up: torch.FloatTensor, + svd_down: torch.FloatTensor, quantized_weight_shape: torch.Size, result_shape: torch.Size, weights_dtype: str, @@ -26,18 +29,26 @@ def conv_int8_matmul( ) -> torch.FloatTensor: return_dtype = input.dtype input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) + if svd_up is not None: + input = input.flatten(0,-2) + if bias is not None: + bias = torch.addmm(bias, torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) + else: + bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) + input, scale = quantize_int8_matmul_input(input, scale) if quantized_weight_shape is not None: weight = unpack_int_symetric(weight, quantized_weight_shape, weights_dtype, dtype=torch.int8) + input, weight = check_mats(input, weight) if groups == 1: - result = torch._int_mm(input, weight) + result = int_mm_func(input, weight) else: weight = weight.view(weight.shape[0], groups, weight.shape[1] // groups) input = input.view(input.shape[0], groups, input.shape[1] // groups) result = [] for i in range(groups): - result.append(torch._int_mm(input[:, i], weight[:, i])) + result.append(int_mm_func(input[:, i], weight[:, i])) result = torch.cat(result, dim=-1) if bias is not None: result = dequantize_symmetric_with_bias(result, scale, bias, return_dtype, mm_output_shape) @@ -55,18 +66,19 @@ def conv_int8_matmul( def quantized_conv_forward_int8_matmul(self, input) -> torch.FloatTensor: if torch.numel(input) / input.shape[2] < 32: - return self._conv_forward(input, self.sdnq_dequantizer(self.weight, skip_quantized_matmul=True), self.bias) + return self._conv_forward(input, self.sdnq_dequantizer(self.weight, self.scale, self.zero_point, self.svd_up, self.svd_down, skip_quantized_matmul=True), self.bias) conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) if self.sdnq_dequantizer.re_quantize_for_matmul: - weight, scale = self.sdnq_dequantizer.re_quantize_matmul(self.weight) + weight, scale = self.sdnq_dequantizer.re_quantize_matmul(self.weight, self.scale, self.zero_point, None, None) quantized_weight_shape = None else: weight = self.weight - scale = self.sdnq_dequantizer.scale + scale = self.scale quantized_weight_shape = getattr(self.sdnq_dequantizer, "quantized_weight_shape", None) return conv_int8_matmul( input, weight, self.bias, - scale, quantized_weight_shape, + scale, self.svd_up, self.svd_down, + quantized_weight_shape, self.sdnq_dequantizer.result_shape, self.sdnq_dequantizer.weights_dtype, self._reversed_padding_repeated_twice, @@ -75,5 +87,4 @@ def quantized_conv_forward_int8_matmul(self, input) -> torch.FloatTensor: ) -if use_torch_compile: - conv_int8_matmul = torch.compile(conv_int8_matmul, fullgraph=True, dynamic=False) +conv_int8_matmul = compile_func(conv_int8_matmul) diff --git a/modules/sdnq/layers/conv/forward.py b/modules/sdnq/layers/conv/forward.py index 44c90b061..2ed3d816f 100644 --- a/modules/sdnq/layers/conv/forward.py +++ b/modules/sdnq/layers/conv/forward.py @@ -75,19 +75,19 @@ def process_conv_input(conv_type, input, reversed_padding_repeated_twice, paddin def quantized_conv_forward(self, input) -> torch.FloatTensor: - return self._conv_forward(input, self.sdnq_dequantizer(self.weight), self.bias) + return self._conv_forward(input, self.sdnq_dequantizer(self.weight, self.scale, self.zero_point, self.svd_up, self.svd_down), self.bias) def quantized_conv_transpose_1d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 1, self.dilation) - return torch.nn.functional.conv_transpose1d(input, self.sdnq_dequantizer(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + return torch.nn.functional.conv_transpose1d(input, self.sdnq_dequantizer(self.weight, self.scale, self.zero_point, self.svd_up, self.svd_down), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) def quantized_conv_transpose_2d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 2, self.dilation) - return torch.nn.functional.conv_transpose2d(input, self.sdnq_dequantizer(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + return torch.nn.functional.conv_transpose2d(input, self.sdnq_dequantizer(self.weight, self.scale, self.zero_point, self.svd_up, self.svd_down), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) def quantized_conv_transpose_3d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 3, self.dilation) - return torch.nn.functional.conv_transpose3d(input, self.sdnq_dequantizer(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + return torch.nn.functional.conv_transpose3d(input, self.sdnq_dequantizer(self.weight, self.scale, self.zero_point, self.svd_up, self.svd_down), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) diff --git a/modules/sdnq/layers/linear/forward.py b/modules/sdnq/layers/linear/forward.py index 20224204c..2371b6abb 100644 --- a/modules/sdnq/layers/linear/forward.py +++ b/modules/sdnq/layers/linear/forward.py @@ -1,7 +1,20 @@ # pylint: disable=relative-beyond-top-level,redefined-builtin,protected-access +from typing import Tuple + import torch +from ...common import use_contiguous_mm # noqa: TID252 + + +def check_mats(input: torch.Tensor, weight: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + input = input.contiguous() + if use_contiguous_mm: + weight = weight.contiguous() + elif weight.is_contiguous(): + weight = weight.t().contiguous().t() + return input, weight + def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor: - return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight), self.bias) + return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight, self.scale, self.zero_point, self.svd_up, self.svd_down), self.bias) diff --git a/modules/sdnq/layers/linear/linear_fp8.py b/modules/sdnq/layers/linear/linear_fp8.py index 5707ad225..2ddd137c1 100644 --- a/modules/sdnq/layers/linear/linear_fp8.py +++ b/modules/sdnq/layers/linear/linear_fp8.py @@ -4,8 +4,9 @@ from typing import Tuple import torch -from ...common import use_torch_compile # noqa: TID252 +from ...common import compile_func # noqa: TID252 from ...dequantizer import quantize_fp8 # noqa: TID252 +from .forward import check_mats def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.Tensor, torch.FloatTensor]: @@ -19,20 +20,29 @@ def fp8_matmul( weight: torch.Tensor, bias: torch.FloatTensor, scale: torch.FloatTensor, + svd_up: torch.FloatTensor, + svd_down: torch.FloatTensor, ) -> torch.FloatTensor: return_dtype = input.dtype output_shape = (*input.shape[:-1], weight.shape[-1]) + if svd_up is not None: + input = input.flatten(0,-2) + svd_bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) input, input_scale = quantize_fp8_matmul_input(input) + input, weight = check_mats(input, weight) if bias is not None and bias.dtype != torch.bfloat16: bias = bias.to(dtype=torch.bfloat16) - return torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=torch.bfloat16).view(output_shape).to(return_dtype) + result = torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=torch.bfloat16) + if svd_up is not None: + result.add_(svd_bias) + result = result.view(output_shape).to(return_dtype) + return result def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: if torch.numel(input) / input.shape[-1] < 32: - return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight, skip_quantized_matmul=True), self.bias) - return fp8_matmul(input, self.weight, self.bias, self.sdnq_dequantizer.scale) + return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight, self.scale, self.zero_point, self.svd_up, self.svd_down, skip_quantized_matmul=True), self.bias) + return fp8_matmul(input, self.weight, self.bias, self.scale, self.svd_up, self.svd_down) -if use_torch_compile: - fp8_matmul = torch.compile(fp8_matmul, fullgraph=True, dynamic=False) +fp8_matmul = compile_func(fp8_matmul) diff --git a/modules/sdnq/layers/linear/linear_fp8_tensorwise.py b/modules/sdnq/layers/linear/linear_fp8_tensorwise.py index aa80251ae..c7978ef0a 100644 --- a/modules/sdnq/layers/linear/linear_fp8_tensorwise.py +++ b/modules/sdnq/layers/linear/linear_fp8_tensorwise.py @@ -4,8 +4,9 @@ from typing import Tuple import torch -from ...common import use_torch_compile # noqa: TID252 +from ...common import compile_func # noqa: TID252 from ...dequantizer import quantize_fp8, dequantize_symmetric, dequantize_symmetric_with_bias # noqa: TID252 +from .forward import check_mats def quantize_fp8_matmul_input_tensorwise(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.Tensor, torch.FloatTensor]: @@ -22,11 +23,20 @@ def fp8_matmul_tensorwise( weight: torch.Tensor, bias: torch.FloatTensor, scale: torch.FloatTensor, + svd_up: torch.FloatTensor, + svd_down: torch.FloatTensor, ) -> torch.FloatTensor: return_dtype = input.dtype output_shape = (*input.shape[:-1], weight.shape[-1]) + if svd_up is not None: + input.flatten(0,-2) + if bias is not None: + bias = torch.addmm(bias, torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) + else: + bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) + input, weight = check_mats(input, weight) if bias is not None: return dequantize_symmetric_with_bias(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, bias, return_dtype, output_shape) else: @@ -35,9 +45,8 @@ def fp8_matmul_tensorwise( def quantized_linear_forward_fp8_matmul_tensorwise(self, input: torch.FloatTensor) -> torch.FloatTensor: if torch.numel(input) / input.shape[-1] < 32: - return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight, skip_quantized_matmul=True), self.bias) - return fp8_matmul_tensorwise(input, self.weight, self.bias, self.sdnq_dequantizer.scale) + return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight, self.scale, self.zero_point, self.svd_up, self.svd_down, skip_quantized_matmul=True), self.bias) + return fp8_matmul_tensorwise(input, self.weight, self.bias, self.scale, self.svd_up, self.svd_down) -if use_torch_compile: - fp8_matmul_tensorwise = torch.compile(fp8_matmul_tensorwise, fullgraph=True, dynamic=False) +fp8_matmul_tensorwise = compile_func(fp8_matmul_tensorwise) diff --git a/modules/sdnq/layers/linear/linear_int8.py b/modules/sdnq/layers/linear/linear_int8.py index 3a1b82c1f..6d7f6f2b8 100644 --- a/modules/sdnq/layers/linear/linear_int8.py +++ b/modules/sdnq/layers/linear/linear_int8.py @@ -4,9 +4,10 @@ from typing import Tuple import torch -from ...common import use_torch_compile # noqa: TID252 +from ...common import compile_func, int_mm_func # noqa: TID252 from ...packed_int import unpack_int_symetric # noqa: TID252 from ...dequantizer import quantize_int8, dequantize_symmetric, dequantize_symmetric_with_bias # noqa: TID252 +from .forward import check_mats def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.CharTensor, torch.FloatTensor]: @@ -23,6 +24,8 @@ def int8_matmul( weight: torch.Tensor, bias: torch.FloatTensor, scale: torch.FloatTensor, + svd_up: torch.FloatTensor, + svd_down: torch.FloatTensor, quantized_weight_shape: torch.Size, weights_dtype: str, ) -> torch.FloatTensor: @@ -30,25 +33,31 @@ def int8_matmul( weight = unpack_int_symetric(weight, quantized_weight_shape, weights_dtype, dtype=torch.int8) return_dtype = input.dtype output_shape = (*input.shape[:-1], weight.shape[-1]) + if svd_up is not None: + input = input.flatten(0,-2) + if bias is not None: + bias = torch.addmm(bias, torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) + else: + bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) input, scale = quantize_int8_matmul_input(input, scale) + input, weight = check_mats(input, weight) if bias is not None: - return dequantize_symmetric_with_bias(torch._int_mm(input, weight), scale, bias, return_dtype, output_shape) + return dequantize_symmetric_with_bias(int_mm_func(input, weight), scale, bias, return_dtype, output_shape) else: - return dequantize_symmetric(torch._int_mm(input, weight), scale, return_dtype, output_shape) + return dequantize_symmetric(int_mm_func(input, weight), scale, return_dtype, output_shape) def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: if torch.numel(input) / input.shape[-1] < 32: - return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight, skip_quantized_matmul=True), self.bias) + return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight, self.scale, self.zero_point, self.svd_up, self.svd_down, skip_quantized_matmul=True), self.bias) if self.sdnq_dequantizer.re_quantize_for_matmul: - weight, scale = self.sdnq_dequantizer.re_quantize_matmul(self.weight) + weight, scale = self.sdnq_dequantizer.re_quantize_matmul(self.weight, self.scale, self.zero_point, None, None) quantized_weight_shape = None else: weight = self.weight - scale = self.sdnq_dequantizer.scale + scale = self.scale quantized_weight_shape = getattr(self.sdnq_dequantizer, "quantized_weight_shape", None) - return int8_matmul(input, weight, self.bias, scale, quantized_weight_shape, self.sdnq_dequantizer.weights_dtype) + return int8_matmul(input, weight, self.bias, scale, self.svd_up, self.svd_down, quantized_weight_shape, self.sdnq_dequantizer.weights_dtype) -if use_torch_compile: - int8_matmul = torch.compile(int8_matmul, fullgraph=True, dynamic=False) +int8_matmul = compile_func(int8_matmul) diff --git a/modules/sdnq/loader.py b/modules/sdnq/loader.py new file mode 100644 index 000000000..88d8140a7 --- /dev/null +++ b/modules/sdnq/loader.py @@ -0,0 +1,192 @@ +import os +import re +import json +import torch +from diffusers.models.modeling_utils import ModelMixin +from .common import use_tensorwise_fp8_matmul, use_contiguous_mm +from .quantizer import SDNQConfig, sdnq_post_load_quant +from .dequantizer import dequantize_symmetric, re_quantize_int8, re_quantize_fp8 + + +def get_module_names(model: ModelMixin) -> list: + modules_names = model._internal_dict.keys() # pylint: disable=protected-access + modules_names = [m for m in modules_names if not m.startswith("_")] + modules_names = [m for m in modules_names if isinstance(getattr(model, m, None), torch.nn.Module)] + modules_names = sorted(set(modules_names)) + return modules_names + + +def unset_config_on_save(quantization_config: SDNQConfig) -> SDNQConfig: + quantization_config.quantization_device = None + quantization_config.return_device = None + quantization_config.non_blocking = False + quantization_config.add_skip_keys = False + return quantization_config + + +def save_sdnq_model(model: ModelMixin, model_path: str, max_shard_size: str = "10GB", is_pipeline: bool = False, sdnq_config: SDNQConfig = None) -> None: + if is_pipeline: + for module_name in get_module_names(model): + module = getattr(model, module_name, None) + if hasattr(module, "config") and hasattr(module.config, "quantization_config") and isinstance(module.config.quantization_config, SDNQConfig): + module.config.quantization_config = unset_config_on_save(module.config.quantization_config) + if hasattr(module, "quantization_config") and isinstance(module.quantization_config, SDNQConfig): + module.quantization_config = unset_config_on_save(module.quantization_config) + else: + if hasattr(model, "config") and hasattr(model.config, "quantization_config") and isinstance(model.config.quantization_config, SDNQConfig): + model.config.quantization_config = unset_config_on_save(model.config.quantization_config) + if hasattr(model, "quantization_config") and isinstance(model.quantization_config, SDNQConfig): + model.quantization_config = unset_config_on_save(model.quantization_config) + + model.save_pretrained(model_path, max_shard_size=max_shard_size) # actual save + + quantization_config_path = os.path.join(model_path, "quantization_config.json") + if sdnq_config is not None: # if provided, save global config + sdnq_config = unset_config_on_save(sdnq_config) + sdnq_config.to_json_file(quantization_config_path) + + if is_pipeline: + for module_name in get_module_names(model): # save per-module config if available + module = getattr(model, module_name, None) + if module is None: + continue + module_quantization_config_path = os.path.join(model_path, module_name, "quantization_config.json") + if hasattr(module, "quantization_config") and isinstance(module.quantization_config, SDNQConfig): + module.quantization_config.to_json_file(module_quantization_config_path) + elif hasattr(module, "config") and hasattr(module.config, "quantization_config") and isinstance(module.config.quantization_config, SDNQConfig): + module.config.quantization_config.to_json_file(module_quantization_config_path) + elif sdnq_config is None: + if hasattr(model, "quantization_config") and isinstance(model.quantization_config, SDNQConfig): + model.quantization_config.to_json_file(quantization_config_path) + elif hasattr(model, "config") and hasattr(model.config, "quantization_config") and isinstance(model.config.quantization_config, SDNQConfig): + model.config.quantization_config.to_json_file(quantization_config_path) + + +def load_sdnq_model(model_path: str, model_cls: ModelMixin = None, file_name: str = None, dtype: torch.dtype = None, device: torch.device = "cpu", dequantize_fp32: bool = None, use_quantized_matmul: bool = None, model_config: dict = None, quantization_config: dict = None) -> ModelMixin: + from accelerate import init_empty_weights + from safetensors.torch import safe_open + + with init_empty_weights(): + if quantization_config is None: + try: + with open(os.path.join(model_path, "quantization_config.json"), "r", encoding="utf-8") as f: + quantization_config = json.load(f) + except Exception: + quantization_config = {} + + if model_config is None: + try: + with open(os.path.join(model_path, "config.json"), "r", encoding="utf-8") as f: + model_config = json.load(f) + except Exception: + model_config = {} + + if model_cls is None: + import transformers + import diffusers + class_name = model_config.get("_class_name", None) or model_config.get("architectures", None) + if isinstance(class_name, list): + class_name = class_name[0] + if class_name is not None: + model_cls = getattr(diffusers, class_name, None) or getattr(transformers, class_name, None) + if model_cls is None: + raise ValueError(f"Cannot determine model class for {model_path}, please provide model_cls argument") + + quantization_config.pop("is_integer", None) + quantization_config.pop("quant_method", None) + quantization_config.pop("quantization_device", None) + quantization_config.pop("return_device", None) + quantization_config.pop("non_blocking", None) + quantization_config.pop("add_skip_keys", None) + + if hasattr(model_cls, "load_config"): + config = model_cls.load_config(model_path) + model = model_cls.from_config(config) + elif hasattr(model_cls, "_from_config"): + config = transformers.AutoConfig.from_pretrained(model_path) + model = model_cls(config) + else: + raise ValueError(f"Dont know how to load model for {model_cls}") + + model = sdnq_post_load_quant(model, add_skip_keys=False, **quantization_config) + + key_mapping = getattr(model, "_checkpoint_conversion_mapping", None) + state_dict = {} + files = [] + + if file_name: + files.append(os.path.join(model_path, file_name)) + else: + all_files = os.listdir(model_path) + files = sorted([os.path.join(model_path, f) for f in all_files if f.endswith(".safetensors")]) + + for fn in files: + with safe_open(fn, framework="pt", device=str(device)) as f: + for key in f.keys(): + new_key = key + if key_mapping: + for pattern, replacement in key_mapping.items(): + new_key, n_replace = re.subn(pattern, replacement, new_key) + if n_replace > 0: + break + state_dict[new_key] = f.get_tensor(key) + + if model.__class__.__name__ in {"T5EncoderModel", "UMT5EncoderModel"} and "encoder.embed_tokens.weight" not in state_dict.keys(): + state_dict["encoder.embed_tokens.weight"] = state_dict["shared.weight"] + + model.load_state_dict(state_dict, assign=True) + del state_dict + + if (dtype is not None) or (dequantize_fp32 is not None) or (use_quantized_matmul is not None): + model = apply_options_to_model(model, dtype=dtype, dequantize_fp32=dequantize_fp32, use_quantized_matmul=use_quantized_matmul) + return model + + +def apply_options_to_model(model, dtype: torch.dtype = None, dequantize_fp32: bool = None, use_quantized_matmul: bool = None): + has_children = list(model.children()) + if not has_children: + return model + for module in model.children(): + if hasattr(module, "sdnq_dequantizer"): + if dtype is not None: + module.sdnq_dequantizer.result_dtype = dtype + + current_scale_dtype = module.svd_up.dtype if module.svd_up is not None else module.scale.dtype + scale_dtype = torch.float32 if dequantize_fp32 is None and current_scale_dtype == torch.float32 else torch.float32 if dequantize_fp32 else module.sdnq_dequantizer.result_dtype + upcast_scale = bool(not use_tensorwise_fp8_matmul and module.sdnq_dequantizer.weights_dtype in {"float8_e4m3fn", "float8_e5m2"} and (use_quantized_matmul or (use_quantized_matmul is None and module.sdnq_dequantizer.use_quantized_matmul))) + + if upcast_scale: + module.scale.data = module.scale.to(dtype=torch.float32) + else: + module.scale.data = module.scale.to(dtype=scale_dtype) + if module.zero_point is not None: + module.zero_point.data = module.zero_point.to(dtype=scale_dtype) + if module.svd_up is not None: + module.svd_up.data = module.svd_up.to(dtype=scale_dtype) + module.svd_down.data = module.svd_down.to(dtype=scale_dtype) + + if use_quantized_matmul is not None and use_quantized_matmul != module.sdnq_dequantizer.use_quantized_matmul: + if module.sdnq_dequantizer.weights_dtype in {"int8", "float8_e4m3fn", "float8_e5m2"}: + if use_quantized_matmul and module.sdnq_dequantizer.re_quantize_for_matmul: + scale_dtype = module.scale.dtype + if module.sdnq_dequantizer.weights_dtype == "int8": + module.weight.data, module.scale.data = re_quantize_int8(dequantize_symmetric(module.weight, module.scale, torch.float32, module.sdnq_dequantizer.result_shape)) + module.scale.data = module.scale.to(dtype=scale_dtype) + else: + is_e5 = bool(module.sdnq_dequantizer.weights_dtype == "float8_e5m2") + module.weight.data, module.scale.data = re_quantize_fp8(dequantize_symmetric(module.weight, module.scale, torch.float32, module.sdnq_dequantizer.result_shape), is_e5=is_e5) + if use_tensorwise_fp8_matmul: + module.scale.data = module.scale.to(dtype=scale_dtype) + elif not module.sdnq_dequantizer.re_quantize_for_matmul: + module.weight.data, module.scale.data = module.weight.t_(), module.scale.t_() + if use_quantized_matmul: + if use_contiguous_mm: + module.weight.data = module.weight.contiguous() + elif module.weight.is_contiguous(): + module.weight.data = module.weight.t_().contiguous().t_() + if module.svd_up is not None: + module.svd_up.data = module.svd_up.t_() + module.svd_down.data = module.svd_down.t_() + module.sdnq_dequantizer.use_quantized_matmul = use_quantized_matmul + module = apply_options_to_model(module, dtype=dtype, dequantize_fp32=dequantize_fp32, use_quantized_matmul=use_quantized_matmul) + return model diff --git a/modules/sdnq/packed_int.py b/modules/sdnq/packed_int.py index 31e8d9214..09a38efbc 100644 --- a/modules/sdnq/packed_int.py +++ b/modules/sdnq/packed_int.py @@ -219,79 +219,64 @@ def unpack_uint4(packed_tensor: torch.ByteTensor, shape: torch.Size) -> torch.By def unpack_uint3(packed_tensor: torch.ByteTensor, shape: torch.Size) -> torch.ByteTensor: - result = torch.cat( - ( - torch.bitwise_and( - torch.cat( - ( - packed_tensor[:, :3], - torch.bitwise_right_shift(packed_tensor[:, :3], 3) - ), - dim=-1 - ), - 7 - ), - torch.bitwise_or( - torch.bitwise_right_shift(packed_tensor[:, :2], 6), - torch.bitwise_and( - torch.stack( - ( - torch.bitwise_right_shift(packed_tensor[:, 2], 4), - torch.bitwise_right_shift(packed_tensor[:, 2], 5), + result = torch.bitwise_and( + torch.cat( + ( + packed_tensor[:, :3], + torch.bitwise_right_shift(packed_tensor[:, :3], 3), + torch.bitwise_or( + torch.bitwise_right_shift(packed_tensor[:, :2], 6), + torch.bitwise_and( + torch.stack( + ( + torch.bitwise_right_shift(packed_tensor[:, 2], 4), + torch.bitwise_right_shift(packed_tensor[:, 2], 5), + ), + dim=-1 ), - dim=-1 + 4 ), - 4 ), ), + dim=-1 ), - dim=-1 + 7 ).view(shape) return result def unpack_uint2(packed_tensor: torch.ByteTensor, shape: torch.Size) -> torch.ByteTensor: - result = torch.cat( - ( - torch.bitwise_and( - torch.stack( - ( - packed_tensor, - torch.bitwise_right_shift(packed_tensor, 2), - torch.bitwise_right_shift(packed_tensor, 4) - ), - dim=-1 - ), - 3 + result = torch.bitwise_and( + torch.stack( + ( + packed_tensor, + torch.bitwise_right_shift(packed_tensor, 2), + torch.bitwise_right_shift(packed_tensor, 4), + torch.bitwise_right_shift(packed_tensor, 6), ), - torch.bitwise_right_shift(packed_tensor, 6).unsqueeze(-1), + dim=-1 ), - dim=-1 + 3 ).view(shape) return result def unpack_uint1(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: - result = torch.cat( - ( - torch.bitwise_and( - torch.stack( - ( - packed_tensor, - torch.bitwise_right_shift(packed_tensor, 1), - torch.bitwise_right_shift(packed_tensor, 2), - torch.bitwise_right_shift(packed_tensor, 3), - torch.bitwise_right_shift(packed_tensor, 4), - torch.bitwise_right_shift(packed_tensor, 5), - torch.bitwise_right_shift(packed_tensor, 6), - ), - dim=-1 - ), - 1 + result = torch.bitwise_and( + torch.stack( + ( + packed_tensor, + torch.bitwise_right_shift(packed_tensor, 1), + torch.bitwise_right_shift(packed_tensor, 2), + torch.bitwise_right_shift(packed_tensor, 3), + torch.bitwise_right_shift(packed_tensor, 4), + torch.bitwise_right_shift(packed_tensor, 5), + torch.bitwise_right_shift(packed_tensor, 6), + torch.bitwise_right_shift(packed_tensor, 7), ), - torch.bitwise_right_shift(packed_tensor, 7).unsqueeze(-1), + dim=-1 ), - dim=-1 + 1 ).view(shape) return result diff --git a/modules/sdnq/quantizer.py b/modules/sdnq/quantizer.py new file mode 100644 index 000000000..29f3b61d8 --- /dev/null +++ b/modules/sdnq/quantizer.py @@ -0,0 +1,762 @@ +# pylint: disable=redefined-builtin,no-member,protected-access + +from typing import Dict, List, Tuple, Optional, Union +from dataclasses import dataclass +from enum import Enum + +import re +import torch +from transformers.quantizers import HfQuantizer +from diffusers.quantizers.base import DiffusersQuantizer +from diffusers.quantizers.quantization_config import QuantizationConfigMixin +from diffusers.utils import get_module_from_name +from modules import devices, shared + +from .common import dtype_dict, module_skip_keys_dict, accepted_weights, use_tensorwise_fp8_matmul, allowed_types, conv_types, conv_transpose_types, use_contiguous_mm +from .dequantizer import dequantizer_dict, dequantize_sdnq_model +from .forward import get_forward_func + + +class QuantizationMethod(str, Enum): + SDNQ = "sdnq" + + +def get_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: Union[int, List[int]], weights_dtype: str) -> Tuple[torch.FloatTensor, torch.FloatTensor]: + zero_point = torch.amin(weight, dim=reduction_axes, keepdims=True) + scale = torch.amax(weight, dim=reduction_axes, keepdims=True).sub_(zero_point).div_(dtype_dict[weights_dtype]["max"] - dtype_dict[weights_dtype]["min"]) + if dtype_dict[weights_dtype]["min"] != 0: + zero_point.sub_(torch.mul(scale, dtype_dict[weights_dtype]["min"])) + return scale, zero_point + + +def get_scale_symmetric(weight: torch.FloatTensor, reduction_axes: Union[int, List[int]], weights_dtype: str) -> torch.FloatTensor: + return torch.amax(weight.abs(), dim=reduction_axes, keepdims=True).div_(dtype_dict[weights_dtype]["max"]) + + +def quantize_weight(weight: torch.FloatTensor, reduction_axes: Union[int, List[int]], weights_dtype: str) -> Tuple[torch.Tensor, torch.FloatTensor, torch.FloatTensor]: + if dtype_dict[weights_dtype]["is_unsigned"]: + scale, zero_point = get_scale_asymmetric(weight, reduction_axes, weights_dtype) + quantized_weight = torch.sub(weight, zero_point).div_(scale) + else: + scale = get_scale_symmetric(weight, reduction_axes, weights_dtype) + quantized_weight = torch.div(weight, scale) + zero_point = None + if dtype_dict[weights_dtype]["is_integer"]: + quantized_weight.round_() + else: + quantized_weight.nan_to_num_() + quantized_weight = quantized_weight.clamp_(dtype_dict[weights_dtype]["min"], dtype_dict[weights_dtype]["max"]).to(dtype_dict[weights_dtype]["torch_dtype"]) + return quantized_weight, scale, zero_point + + +def apply_svdquant(weight: torch.FloatTensor, rank: int = 32, niter: int = 8) -> Tuple[torch.FloatTensor, torch.FloatTensor, torch.FloatTensor]: + reshape_weight = False + if weight.ndim > 2: # convs + reshape_weight = True + weight_shape = weight.shape + weight = weight.flatten(1,-1) + U, S, svd_down = torch.svd_lowrank(weight, q=rank, niter=niter) + svd_up = torch.mul(U, S.unsqueeze(0)) + svd_down = svd_down.t_() + weight = weight.sub_(torch.mm(svd_up, svd_down)) + if reshape_weight: + weight = weight.unflatten(-1, (*weight_shape[1:],)) # pylint: disable=possibly-used-before-assignment + return weight, svd_up, svd_down + + +def check_param_name_in(param_name: str, param_list: List[str]) -> bool: + split_param_name = param_name.split(".") + for param in param_list: + if param.startswith("."): + if param_name.startswith(param[1:]): + return True + else: + continue + if ( + param_name == param + or param in split_param_name + or ("*" in param and re.match(param.replace(".*", "\\.*").replace("*", ".*"), param_name)) + ): + return True + return False + + +def get_minimum_dtype(weights_dtype: str, param_name: str, modules_dtype_dict: Dict[str, List[str]]): + if len(modules_dtype_dict.keys()) > 0: + for key, value in modules_dtype_dict.items(): + if check_param_name_in(param_name, value): + key = key.lower() + if key in {"8bit", "8bits"}: + if dtype_dict[weights_dtype]["num_bits"] != 8: + return "int8" + elif key.startswith("minimum_"): + minimum_bits_str = key.removeprefix("minimum_").removesuffix("bits").removesuffix("bit") + if minimum_bits_str.startswith("uint"): + is_unsigned = True + minimum_bits_str = minimum_bits_str.removeprefix("uint") + else: + is_unsigned = False + minimum_bits_str = minimum_bits_str.removeprefix("int") + minimum_bits = int(minimum_bits_str) + if dtype_dict[weights_dtype]["num_bits"] < minimum_bits: + if is_unsigned or minimum_bits <= 4: + return "uint" + minimum_bits_str + else: + return "int" + minimum_bits_str + else: + return key + return weights_dtype + + +def add_module_skip_keys(model, modules_to_not_convert: List[str] = None, modules_dtype_dict: Dict[str, List[str]] = None): + if modules_to_not_convert is None: + modules_to_not_convert = [] + if modules_dtype_dict is None: + modules_dtype_dict = {} + if getattr(model, "_keep_in_fp32_modules", None) is not None: + modules_to_not_convert.extend(model._keep_in_fp32_modules) # pylint: disable=protected-access + + skip_key_list = module_skip_keys_dict.get(model.__class__.__name__, None) + if skip_key_list is not None: + modules_to_not_convert.extend(skip_key_list[0]) + for key, value in skip_key_list[1].items(): + if key in modules_dtype_dict.keys(): + modules_dtype_dict[key].extend(value) + else: + modules_dtype_dict[key] = value + elif getattr(model, "_skip_layerwise_casting_patterns", None) is not None: + modules_to_not_convert.extend(model._skip_layerwise_casting_patterns) # pylint: disable=protected-access + + # dedupe + modules_to_not_convert = list(set(modules_to_not_convert)) + for key, value in modules_dtype_dict.items(): + modules_dtype_dict[key] = list(set(value)) + + return model, modules_to_not_convert, modules_dtype_dict + + +@devices.inference_context() +def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, svd_rank=32, svd_steps=8, use_svd=False, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, dequantize_fp32=False, non_blocking=False, quantization_device=None, return_device=None, param_name=None): # pylint: disable=unused-argument + layer_class_name = layer.__class__.__name__ + if layer_class_name in allowed_types: + num_of_groups = 1 + is_conv_type = False + is_conv_transpose_type = False + is_linear_type = False + result_shape = None + original_shape = layer.weight.shape + if torch_dtype is None: + torch_dtype = layer.weight.dtype + + if layer_class_name in conv_types: + if not quant_conv: + return layer + if dtype_dict[weights_dtype]["num_bits"] < 4: + weights_dtype = "uint4" + is_conv_type = True + reduction_axes = 1 + output_channel_size, channel_size = layer.weight.shape[:2] + group_channel_size = channel_size // layer.groups + use_quantized_matmul = False + if use_quantized_matmul_conv: + use_quantized_matmul = group_channel_size >= 32 and output_channel_size >= 32 + if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: + use_quantized_matmul = output_channel_size % 16 == 0 and group_channel_size % 16 == 0 + if use_quantized_matmul and dtype_dict[weights_dtype]["num_bits"] == 8: + result_shape = layer.weight.shape + layer.weight.data = layer.weight.flatten(1,-1) + reduction_axes = -1 + elif layer_class_name in conv_transpose_types: + if not quant_conv: + return layer + if dtype_dict[weights_dtype]["num_bits"] < 4: + weights_dtype = "uint4" + is_conv_transpose_type = True + reduction_axes = 0 + channel_size, output_channel_size = layer.weight.shape[:2] + use_quantized_matmul = False + else: + is_linear_type = True + reduction_axes = -1 + try: + output_channel_size, channel_size = layer.weight.shape + except Exception as e: + raise ValueError(f"SDNQ: param_name={param_name} layer_class_name={layer_class_name} layer_weight_shape={layer.weight.shape} weights_dtype={weights_dtype} unsupported") from e + if use_quantized_matmul: + use_quantized_matmul = channel_size >= 32 and output_channel_size >= 32 + if use_quantized_matmul: + if dtype_dict[weights_dtype]["is_integer"]: + use_quantized_matmul = output_channel_size % 8 == 0 and channel_size % 8 == 0 + else: + use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 + + layer.weight.requires_grad = False + if return_device is None: + return_device = layer.weight.device + if quantization_device is not None: + layer.weight.data = layer.weight.to(quantization_device, non_blocking=non_blocking) + if layer.weight.dtype != torch.float32: + layer.weight.data = layer.weight.to(dtype=torch.float32) + + if use_svd: + layer.weight.data, svd_up, svd_down = apply_svdquant(layer.weight, rank=svd_rank, niter=svd_steps) + if use_quantized_matmul: + svd_up = svd_up.t_() + svd_down = svd_down.t_() + else: + svd_up, svd_down = None, None + + if group_size == 0: + if use_quantized_matmul and dtype_dict[weights_dtype]["num_bits"] >= 6: + group_size = -1 + elif is_linear_type: + group_size = 2 ** ((2 if not use_svd else 3) + dtype_dict[weights_dtype]["num_bits"]) + else: + group_size = 2 ** ((1 if not use_svd else 2) + dtype_dict[weights_dtype]["num_bits"]) + elif use_quantized_matmul and dtype_dict[weights_dtype]["num_bits"] == 8: + group_size = -1 # override user value, re-quantizing 8bit into 8bit is pointless + elif group_size != -1 and not is_linear_type: + group_size = max(group_size // 2, 1) + + if group_size > 0: + if group_size >= channel_size: + group_size = channel_size + num_of_groups = 1 + else: + num_of_groups = channel_size // group_size + while num_of_groups * group_size != channel_size: # find something divisible + num_of_groups -= 1 + if num_of_groups <= 1: + group_size = channel_size + num_of_groups = 1 + break + group_size = channel_size // num_of_groups + group_size = int(group_size) + num_of_groups = int(num_of_groups) + + if num_of_groups > 1: + if result_shape is None: + result_shape = layer.weight.shape + new_shape = list(result_shape) + if is_conv_type: + # output_channel_size, channel_size, X, X + # output_channel_size, num_of_groups, group_size, X, X + new_shape[1] = group_size + new_shape.insert(1, num_of_groups) + reduction_axes = 2 + elif is_conv_transpose_type: + #channel_size, output_channel_size, X, X + #num_of_groups, group_size, output_channel_size, X, X + new_shape[0] = group_size + new_shape.insert(0, num_of_groups) + reduction_axes = 1 + elif is_linear_type: + # output_channel_size, channel_size + # output_channel_size, num_of_groups, group_size + last_dim_index = layer.weight.ndim + new_shape[last_dim_index - 1 : last_dim_index] = (num_of_groups, group_size) + layer.weight.data = layer.weight.reshape(new_shape) + + layer.weight.data, scale, zero_point = quantize_weight(layer.weight, reduction_axes, weights_dtype) + if not dequantize_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): + scale = scale.to(dtype=torch_dtype) + if zero_point is not None: + zero_point = zero_point.to(dtype=torch_dtype) + if svd_up is not None: + svd_up = svd_up.to(dtype=torch_dtype) + svd_down = svd_down.to(dtype=torch_dtype) + + re_quantize_for_matmul = (num_of_groups > 1 or zero_point is not None) + if use_quantized_matmul and not re_quantize_for_matmul: + scale.t_() + layer.weight.t_() + if use_contiguous_mm: + layer.weight.data = layer.weight.contiguous() + elif layer.weight.is_contiguous(): + layer.weight.data = layer.weight.t_().contiguous().t_() + if not use_tensorwise_fp8_matmul and not dtype_dict[weights_dtype]["is_integer"]: + scale = scale.to(dtype=torch.float32) + + scale = scale.to(return_device, non_blocking=non_blocking) + layer.scale = torch.nn.Parameter(scale, requires_grad=False) + if zero_point is not None: + zero_point = zero_point.to(return_device, non_blocking=non_blocking) + layer.zero_point = torch.nn.Parameter(zero_point, requires_grad=False) + else: + layer.zero_point = None + if svd_up is not None: + svd_up = svd_up.to(return_device, non_blocking=non_blocking) + svd_down = svd_down.to(return_device, non_blocking=non_blocking) + layer.svd_up = torch.nn.Parameter(svd_up, requires_grad=False) + layer.svd_down = torch.nn.Parameter(svd_down, requires_grad=False) + else: + layer.svd_up, layer.svd_down = None, None + + layer.sdnq_dequantizer = dequantizer_dict[weights_dtype]( + quantized_weight_shape=layer.weight.shape, + result_dtype=torch_dtype, + result_shape=result_shape, + original_shape=original_shape, + weights_dtype=weights_dtype, + use_quantized_matmul=use_quantized_matmul, + re_quantize_for_matmul=re_quantize_for_matmul, + ) + layer.weight.data = layer.sdnq_dequantizer.pack_weight(layer.weight).to(return_device, non_blocking=non_blocking) + + layer.forward = get_forward_func(layer_class_name, use_quantized_matmul, dtype_dict[weights_dtype]["is_integer"], use_tensorwise_fp8_matmul) + layer.forward = layer.forward.__get__(layer, layer.__class__) + return layer + + +def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, svd_rank=32, svd_steps=8, use_svd=False, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, dequantize_fp32=False, non_blocking=False, quantization_device=None, return_device=None, modules_to_not_convert: List[str] = None, modules_dtype_dict: Dict[str, List[str]] = None, full_param_name="", op=None): # pylint: disable=unused-argument + has_children = list(model.children()) + if not has_children: + return model + if modules_to_not_convert is None: + modules_to_not_convert = [] + if modules_dtype_dict is None: + modules_dtype_dict = {} + for param_name, module in model.named_children(): + if param_name == "sdnq_dequantizer": + continue + if full_param_name: + param_name = full_param_name + "." + param_name + if hasattr(module, "weight") and module.weight is not None: + param_name = param_name + ".weight" + if check_param_name_in(param_name, modules_to_not_convert): + continue + layer_class_name = module.__class__.__name__ + if layer_class_name in allowed_types: + if (layer_class_name in conv_types or layer_class_name in conv_transpose_types) and not quant_conv: + continue + else: + continue + + weights_dtype = get_minimum_dtype(weights_dtype, param_name, modules_dtype_dict) + module = sdnq_quantize_layer( + module, + weights_dtype=weights_dtype, + torch_dtype=torch_dtype, + group_size=group_size, + svd_rank=svd_rank, + svd_steps=svd_steps, + use_svd=use_svd, + quant_conv=quant_conv, + use_quantized_matmul=use_quantized_matmul, + use_quantized_matmul_conv=use_quantized_matmul_conv, + dequantize_fp32=dequantize_fp32, + non_blocking=non_blocking, + quantization_device=quantization_device, + return_device=return_device, + param_name=param_name, + ) + module = apply_sdnq_to_module( + module, + weights_dtype=weights_dtype, + torch_dtype=torch_dtype, + group_size=group_size, + svd_rank=svd_rank, + svd_steps=svd_steps, + use_svd=use_svd, + quant_conv=quant_conv, + use_quantized_matmul=use_quantized_matmul, + use_quantized_matmul_conv=use_quantized_matmul_conv, + dequantize_fp32=dequantize_fp32, + non_blocking=non_blocking, + quantization_device=quantization_device, + return_device=return_device, + modules_to_not_convert=modules_to_not_convert, + modules_dtype_dict=modules_dtype_dict, + full_param_name=param_name, + op=op, + ) + return model + + +def sdnq_post_load_quant( + model, + weights_dtype="int8", + torch_dtype: torch.dtype = None, + group_size: int = 0, + svd_rank: int = 32, + svd_steps: int = 8, + use_svd: bool = False, + quant_conv: bool = False, + use_quantized_matmul: bool = False, + use_quantized_matmul_conv: bool = False, + dequantize_fp32: bool = False, + non_blocking: bool = False, + add_skip_keys:bool = True, + quantization_device: torch.device = None, + return_device: torch.device = None, + modules_to_not_convert: List[str] = None, + modules_dtype_dict: Dict[str, List[str]] = None, + op=None, +): + if add_skip_keys: + model, modules_to_not_convert, modules_dtype_dict = add_module_skip_keys(model, modules_to_not_convert, modules_dtype_dict) + + model.eval() + model = apply_sdnq_to_module( + model, + weights_dtype=weights_dtype, + torch_dtype=torch_dtype, + group_size=group_size, + svd_rank=svd_rank, + svd_steps=svd_steps, + use_svd=use_svd, + quant_conv=quant_conv, + use_quantized_matmul=use_quantized_matmul, + use_quantized_matmul_conv=use_quantized_matmul_conv, + dequantize_fp32=dequantize_fp32, + non_blocking=non_blocking, + quantization_device=quantization_device, + return_device=return_device, + modules_to_not_convert=modules_to_not_convert, + modules_dtype_dict=modules_dtype_dict.copy(), + op=op, + ) + model.quantization_config = SDNQConfig( + weights_dtype=weights_dtype, + group_size=group_size, + svd_rank=svd_rank, + svd_steps=svd_steps, + use_svd=use_svd, + quant_conv=quant_conv, + use_quantized_matmul=use_quantized_matmul, + use_quantized_matmul_conv=use_quantized_matmul_conv, + dequantize_fp32=dequantize_fp32, + non_blocking=non_blocking, + quantization_device=quantization_device, + return_device=return_device, + modules_to_not_convert=modules_to_not_convert, + modules_dtype_dict=modules_dtype_dict.copy(), + ) + + if hasattr(model, "config"): + try: + model.config.quantization_config = model.quantization_config + model.config["quantization_config"] = model.quantization_config.to_dict() + except Exception: + pass + model.quantization_method = QuantizationMethod.SDNQ + + return model + + +class SDNQQuantizer(DiffusersQuantizer, HfQuantizer): + r""" + Diffusers and Transformers Quantizer for SDNQ + """ + + requires_parameters_quantization = True + use_keep_in_fp32_modules = True + requires_calibration = False + required_packages = None + torch_dtype = None + + def check_if_quantized_param( + self, + model, + param_value: "torch.Tensor", + param_name: str, + *args, **kwargs, # pylint: disable=unused-argument + ): + if self.pre_quantized: + layer, _tensor_name = get_module_from_name(model, param_name) + if hasattr(layer, "sdnq_dequantizer"): + return True + elif param_name.endswith(".weight"): + if not check_param_name_in(param_name, self.quantization_config.modules_to_not_convert): + layer_class_name = get_module_from_name(model, param_name)[0].__class__.__name__ + if layer_class_name in allowed_types: + if layer_class_name in conv_types or layer_class_name in conv_transpose_types: + if self.quantization_config.quant_conv: + return True + else: + return True + if param_value is not None and param_value.device.type == "cpu": + with devices.inference_context(): + param_value.data = param_value.clone() # safetensors is unable to release the cpu memory without this + return False + + def check_quantized_param(self, *args, **kwargs) -> bool: + """ + needed for transformers compatibilty, returns self.check_if_quantized_param + """ + return self.check_if_quantized_param(*args, **kwargs) + + def param_needs_quantization(self, model, param_name: str, *args, **kwargs) -> bool: + """ + needed for transformers compatibilty, returns self.check_if_quantized_param + """ + return self.check_if_quantized_param(model, None, param_name, *args, **kwargs) + + @devices.inference_context() + def create_quantized_param( # pylint: disable=arguments-differ + self, + model, + param_value: torch.FloatTensor, + param_name: str, + target_device: torch.device, + *args, **kwargs, # pylint: disable=unused-argument + ): + if self.pre_quantized: + layer, tensor_name = get_module_from_name(model, param_name) + if param_value is not None: + return_dtype = param_value.dtype if tensor_name == "weight" else torch.float32 if self.quantization_config.dequantize_fp32 else self.torch_dtype if self.torch_dtype is not None else param_value.dtype + if param_value.dtype == return_dtype and devices.same_device(param_value.device, target_device): + param_value = param_value.clone() + else: + param_value = param_value.to(target_device, dtype=return_dtype) + param_value = torch.nn.Parameter(param_value, requires_grad=False) + setattr(layer, tensor_name, param_value) + return + + torch_dtype = param_value.dtype if self.torch_dtype is None else self.torch_dtype + weights_dtype = get_minimum_dtype(self.quantization_config.weights_dtype, param_name, self.quantization_config.modules_dtype_dict) + + if self.quantization_config.return_device is not None: + return_device = self.quantization_config.return_device + else: + return_device = target_device + + if self.quantization_config.quantization_device is not None: + target_device = self.quantization_config.quantization_device + + if param_value.dtype == torch.float32 and devices.same_device(param_value.device, target_device): + param_value = param_value.clone() + else: + param_value = param_value.to(target_device, non_blocking=self.quantization_config.non_blocking).to(dtype=torch.float32) + + layer, _ = get_module_from_name(model, param_name) + layer.weight = torch.nn.Parameter(param_value, requires_grad=False) + layer = sdnq_quantize_layer( + layer, + weights_dtype=weights_dtype, + torch_dtype=torch_dtype, + group_size=self.quantization_config.group_size, + svd_rank=self.quantization_config.svd_rank, + svd_steps=self.quantization_config.svd_steps, + use_svd=self.quantization_config.use_svd, + quant_conv=self.quantization_config.quant_conv, + use_quantized_matmul=self.quantization_config.use_quantized_matmul, + use_quantized_matmul_conv=self.quantization_config.use_quantized_matmul_conv, + dequantize_fp32=self.quantization_config.dequantize_fp32, + non_blocking=self.quantization_config.non_blocking, + quantization_device=None, + return_device=return_device, + param_name=param_name, + ) + + def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: + max_memory = {key: val * 0.80 for key, val in max_memory.items()} + return max_memory + + def adjust_target_dtype(self, target_dtype: torch.dtype) -> torch.dtype: # pylint: disable=unused-argument,arguments-renamed + return dtype_dict[self.quantization_config.weights_dtype]["target_dtype"] + + def update_torch_dtype(self, torch_dtype: torch.dtype = None) -> torch.dtype: + self.torch_dtype = torch_dtype + return torch_dtype + + def _process_model_before_weight_loading( # pylint: disable=arguments-differ + self, + model, + device_map, # pylint: disable=unused-argument + keep_in_fp32_modules: List[str] = None, + **kwargs, # pylint: disable=unused-argument + ): + if self.pre_quantized: + from accelerate import init_empty_weights + self.quantization_config.quantization_device = None + self.quantization_config.return_device = None + self.quantization_config.non_blocking = False + self.quantization_config.add_skip_keys = False + + quantization_config_dict = self.quantization_config.to_dict() + quantization_config_dict.pop("is_integer", None) + quantization_config_dict.pop("quant_method", None) + quantization_config_dict.pop("quantization_device", None) + quantization_config_dict.pop("return_device", None) + quantization_config_dict.pop("non_blocking", None) + quantization_config_dict.pop("add_skip_keys", None) + with init_empty_weights(): + model = sdnq_post_load_quant(model, add_skip_keys=False, **quantization_config_dict) + + if self.quantization_config.add_skip_keys: + if keep_in_fp32_modules is not None: + self.quantization_config.modules_to_not_convert.extend(keep_in_fp32_modules) + model, self.quantization_config.modules_to_not_convert, self.quantization_config.modules_dtype_dict = add_module_skip_keys( + model, self.quantization_config.modules_to_not_convert, self.quantization_config.modules_dtype_dict + ) + if hasattr(model, "config"): + try: + model.config.quantization_config = self.quantization_config + model.config["quantization_config"] = self.quantization_config.to_dict() + except Exception: + pass + model.quantization_config = self.quantization_config + + def _process_model_after_weight_loading(self, model, **kwargs): # pylint: disable=unused-argument + if shared.opts.diffusers_offload_mode != "none": + model = model.to(devices.cpu) + devices.torch_gc(force=True, reason="sdnq") + return model + + def get_accelerator_warm_up_factor(self): + return 32 // dtype_dict[self.quantization_config.weights_dtype]["num_bits"] + + def get_cuda_warm_up_factor(self): + """ + needed for transformers compatibilty, returns self.get_accelerator_warm_up_factor + """ + return self.get_accelerator_warm_up_factor() + + def _dequantize(self, model): + model = dequantize_sdnq_model(model) + if hasattr(model, "quantization_method"): + del model.quantization_method + if hasattr(model, "quantization_config"): + del model.quantization_config + if hasattr(model, "config") and hasattr(model.config, "quantization_config"): + del model.config.quantization_config + return model + + def is_serializable(self, *args, **kwargs) -> bool: # pylint: disable=unused-argument, invalid-overridden-method + return True + + @property + def is_trainable(self): + return False + + @property + def is_compileable(self): + return True + + +@dataclass +class SDNQConfig(QuantizationConfigMixin): + """ + This is a wrapper class about all possible attributes and features that you can play with a model that has been + loaded using `sdnq`. + + Args: + weights_dtype (`str`, *optional*, defaults to `"int8"`): + The target dtype for the weights after quantization. Supported values are: + ("int8", "int7", "int6", "int5", "int4", "int3", "int2", "uint8", "uint7", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz") + group_size (`int`, *optional*, defaults to `0`): + Used to decide how many elements of a tensor will share the same quantization group. + group_size = 0 will automatically select a group size based on weights_dtype. + svd_rank (`int`, *optional*, defaults to `32`): + The rank size used for the SVDQuant algorithm. + svd_steps (`int`, *optional*, defaults to `8`): + The number of iterations to use in svd lowrank estimation. + use_svd (`bool`, *optional*, defaults to `False`): + Enabling this option will use SVDQuant algorithm on top of SDNQ quantization. + quant_conv (`bool`, *optional*, defaults to `False`): + Enabling this option will quantize the convolutional layers in UNet models too. + use_quantized_matmul (`bool`, *optional*, defaults to `False`): + Enabling this option will use quantized INT8 or FP8 MatMul instead of BF16 / FP16. + use_quantized_matmul_conv (`bool`, *optional*, defaults to `False`): + Same as use_quantized_matmul_conv but for the convolutional layers with UNets like SDXL. + dequantize_fp32 (`bool`, *optional*, defaults to `False`): + Enabling this option will use FP32 on the dequantization step. + non_blocking (`bool`, *optional*, defaults to `False`): + Enabling this option will use non blocking ops when moving layers between the quantization device and the return device. + add_skip_keys (`bool`, *optional*, defaults to `True`): + Disabling this option won't add model specific modules_to_not_convert and modules_dtype_dict keys. + quantization_device (`torch.device`, *optional*, defaults to `None`): + Used to set which device will be used for the quantization calculation on model load. + return_device (`torch.device`, *optional*, defaults to `None`): + Used to set which device will the quantized weights be sent back to. + modules_to_not_convert (`list`, *optional*, default to `None`): + The list of modules to not quantize, useful for quantizing models that explicitly require to have some + modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers). + modules_dtype_dict (`dict`, *optional*, default to `None`): + The dict of dtypes and list of modules, useful for quantizing some modules with a different dtype. + """ + + def __init__( # pylint: disable=super-init-not-called + self, + weights_dtype: str = "int8", + group_size: int = 0, + svd_rank: int = 32, + svd_steps: int = 8, + use_svd: bool = False, + quant_conv: bool = False, + use_quantized_matmul: bool = False, + use_quantized_matmul_conv: bool = False, + dequantize_fp32: bool = False, + non_blocking: bool = False, + add_skip_keys: bool = True, + quantization_device: Optional[torch.device] = None, + return_device: Optional[torch.device] = None, + modules_to_not_convert: Optional[List[str]] = None, + modules_dtype_dict: Optional[Dict[str, List[str]]] = None, + **kwargs, # pylint: disable=unused-argument + ): + self.weights_dtype = weights_dtype + self.quant_method = QuantizationMethod.SDNQ + self.group_size = group_size + self.svd_rank = svd_rank + self.svd_steps = svd_steps + self.use_svd = use_svd + self.quant_conv = quant_conv + self.use_quantized_matmul = use_quantized_matmul + self.use_quantized_matmul_conv = use_quantized_matmul_conv + self.dequantize_fp32 = dequantize_fp32 + self.non_blocking = non_blocking + self.add_skip_keys = add_skip_keys + self.quantization_device = quantization_device + self.return_device = return_device + self.modules_to_not_convert = modules_to_not_convert + self.modules_dtype_dict = modules_dtype_dict + self.post_init() + self.is_integer = dtype_dict[self.weights_dtype]["is_integer"] + + def post_init(self): + r""" + Safety checker that arguments are correct + """ + if self.weights_dtype not in accepted_weights: + raise ValueError(f"SDNQ only support weights in {accepted_weights} but found {self.weights_dtype}") + + if self.modules_to_not_convert is None: + self.modules_to_not_convert = [] + elif isinstance(self.modules_to_not_convert, str): + self.modules_to_not_convert = [self.modules_to_not_convert] + elif isinstance(self.modules_to_not_convert, tuple): + self.modules_to_not_convert = list(self.modules_to_not_convert) + elif not isinstance(self.modules_to_not_convert, list): + raise ValueError(f"modules_to_not_convert must be a list but got {type(self.modules_to_not_convert)}") + + if self.modules_dtype_dict is None: + self.modules_dtype_dict = {} + elif not isinstance(self.modules_dtype_dict, dict): + raise ValueError(f"modules_dtype_dict must be a dict but got {type(self.modules_dtype_dict)}") + elif len(self.modules_dtype_dict.keys()) > 0: + for key, value in self.modules_dtype_dict.items(): + if isinstance(value, str): + value = [value] + self.modules_dtype_dict[key] = value + elif isinstance(value, tuple): + value = list(value) + self.modules_dtype_dict[key] = value + if not isinstance(key, str) or not isinstance(value, list): + raise ValueError(f"modules_dtype_dict must be a dictionary of strings and lists but got {type(key)} and {type(value)}") + + def to_dict(self): + dct = self.__dict__.copy() # make serializable + dct["quantization_device"] = str(dct["quantization_device"]) if dct["quantization_device"] is not None else None + dct["return_device"] = str(dct["return_device"]) if dct["return_device"] is not None else None + return dct + + +import diffusers.quantizers.auto # noqa: E402,RUF100 # pylint: disable=wrong-import-order +diffusers.quantizers.auto.AUTO_QUANTIZER_MAPPING["sdnq"] = SDNQQuantizer +diffusers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig + +import transformers.quantizers.auto # noqa: E402,RUF100 # pylint: disable=wrong-import-order +transformers.quantizers.auto.AUTO_QUANTIZER_MAPPING["sdnq"] = SDNQQuantizer +transformers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig diff --git a/modules/sdnq/triton_mm.py b/modules/sdnq/triton_mm.py new file mode 100644 index 000000000..d15294c3e --- /dev/null +++ b/modules/sdnq/triton_mm.py @@ -0,0 +1,122 @@ +""" +Modified from Triton MatMul example. +PyTorch torch._int_mm is broken on backward pass with Nvidia. +AMD RDNA2 doesn't support torch._int_mm, so we use int_mm via Triton. +""" + +import torch + +import triton +import triton.language as tl + + +def get_autotune_config(): + if triton.runtime.driver.active.get_current_target().backend == "cuda": + return [ + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8}, num_stages=3, num_warps=8), + triton.Config({'BLOCK_SIZE_M': 64, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 64, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 64, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 32, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 64, 'BLOCK_SIZE_N': 32, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=5, num_warps=2), + triton.Config({'BLOCK_SIZE_M': 32, 'BLOCK_SIZE_N': 64, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=5, num_warps=2), + # + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=3, num_warps=8), + triton.Config({'BLOCK_SIZE_M': 64, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 64, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 64, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 32, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 256, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=3, num_warps=8), + triton.Config({'BLOCK_SIZE_M': 256, 'BLOCK_SIZE_N': 64, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=4), + ] + else: + return [ + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=8), + triton.Config({'BLOCK_SIZE_M': 64, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 64, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 64, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 32, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 64, 'BLOCK_SIZE_N': 32, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=2), + triton.Config({'BLOCK_SIZE_M': 32, 'BLOCK_SIZE_N': 64, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=2), + # + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=8), + triton.Config({'BLOCK_SIZE_M': 64, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 64, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 64, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 32, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=4), + triton.Config({'BLOCK_SIZE_M': 256, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=8), + triton.Config({'BLOCK_SIZE_M': 256, 'BLOCK_SIZE_N': 64, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=4), + ] + + +@triton.autotune(configs=get_autotune_config(), key=['M', 'N', 'K', 'stride_bk']) +@triton.jit +def int_mm_kernel( + a_ptr, b_ptr, c_ptr, + M, N, K, + stride_am, stride_ak, + stride_bk, stride_bn, + stride_cm, stride_cn, + BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, # + GROUP_SIZE_M: tl.constexpr +): + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + tl.assume(pid_m >= 0) + tl.assume(pid_n >= 0) + tl.assume(stride_am > 0) + tl.assume(stride_ak > 0) + tl.assume(stride_bn > 0) + tl.assume(stride_bk > 0) + tl.assume(stride_cm > 0) + tl.assume(stride_cn > 0) + + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = b_ptr + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.int32) + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) + accumulator = tl.dot(a, b, accumulator, out_dtype=tl.int32) + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, accumulator, mask=c_mask) + + +def int_mm(a, b): + assert a.shape[1] == b.shape[0], "Incompatible dimensions" + assert a.is_contiguous(), "Matrix A must be contiguous" + M, K = a.shape + K, N = b.shape + c = torch.empty((M, N), device=a.device, dtype=torch.int32) + def grid(META): + return (triton.cdiv(M, META['BLOCK_SIZE_M']) * triton.cdiv(N, META['BLOCK_SIZE_N']), ) + int_mm_kernel[grid]( + a, b, c, + M, N, K, + a.stride(0), a.stride(1), + b.stride(0), b.stride(1), + c.stride(0), c.stride(1), + ) + return c diff --git a/modules/seedvr/__init__.py b/modules/seedvr/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/modules/seedvr/config_3b.yaml b/modules/seedvr/config_3b.yaml new file mode 100644 index 000000000..711d4ec71 --- /dev/null +++ b/modules/seedvr/config_3b.yaml @@ -0,0 +1,97 @@ +__object__: + path: projects.video_diffusion_sr.train + name: VideoDiffusionTrainer + +dit: + model: + __object__: + path: + - "SeedVR2_VideoUpscaler.src.models.dit_v2.nadit" + - "SeedVR2_VideoUpscaler.src.models.dit_v2.nadit" + - "modules.seedvr.src.models.dit_v2.nadit" + name: "NaDiT" + args: "as_params" + vid_in_channels: 33 + vid_out_channels: 16 + vid_dim: 2560 + vid_out_norm: fusedrms + txt_in_dim: 5120 + txt_in_norm: fusedln + txt_dim: ${.vid_dim} + emb_dim: ${eval:'6 * ${.vid_dim}'} + heads: 20 + head_dim: 128 # llm-like + expand_ratio: 4 + norm: fusedrms + norm_eps: 1.0e-05 + ada: single + qk_bias: False + qk_norm: fusedrms + patch_size: [1, 2, 2] + num_layers: 32 # llm-like + mm_layers: 10 + mlp_type: swiglu + msa_type: None + block_type: ${eval:'${.num_layers} * ["mmdit_sr"]'} # space-full + window: ${eval:'${.num_layers} * [(4,3,3)]'} # space-full + window_method: ${eval:'${.num_layers} // 2 * ["720pwin_by_size_bysize","720pswin_by_size_bysize"]'} # space-full + rope_type: mmrope3d + rope_dim: 128 + compile: False + gradient_checkpoint: True + fsdp: + sharding_strategy: _HYBRID_SHARD_ZERO2 + +ema: + decay: 0.9998 + +vae: + model: + __object__: + path: + - "SeedVR2_VideoUpscaler.src.models.video_vae_v3.modules.attn_video_vae" + - "SeedVR2_VideoUpscaler.src.models.video_vae_v3.modules.attn_video_vae" + - "modules.seedvr.src.models.video_vae_v3.modules.attn_video_vae" + name: "VideoAutoencoderKLWrapper" + args: "as_params" + freeze_encoder: False + gradient_checkpoint: True # Disabled to prevent VRAM leaks in inference + slicing: + split_size: 4 + memory_device: same + memory_limit: + conv_max_mem: 0.5 + norm_max_mem: 0.5 + checkpoint: ema_vae_fp16.safetensors + scaling_factor: 0.9152 + compile: False + grouping: False + dtype: float16 + +diffusion: + schedule: + type: lerp + T: 1000.0 + sampler: + type: euler + prediction_type: v_lerp + timesteps: + training: + type: logitnormal + loc: 0.0 + scale: 1.0 + sampling: + type: uniform_trailing + steps: 50 + transform: True + loss: + type: v_lerp + cfg: + scale: 7.5 + rescale: 0 + +condition: + i2v: 0.0 + v2v: 0.0 + sr: 1.0 + noise_scale: 0.25 diff --git a/modules/seedvr/config_7b.yaml b/modules/seedvr/config_7b.yaml new file mode 100644 index 000000000..0e5cb146c --- /dev/null +++ b/modules/seedvr/config_7b.yaml @@ -0,0 +1,94 @@ +__object__: + path: projects.video_diffusion_sr.train + name: VideoDiffusionTrainer + +dit: + model: + __object__: + path: + - "SeedVR2_VideoUpscaler.src.models.dit.nadit" + - "SeedVR2_VideoUpscaler.src.models.dit.nadit" + - "modules.seedvr.src.models.dit.nadit" + name: "NaDiT" + args: "as_params" + vid_in_channels: 33 + vid_out_channels: 16 + vid_dim: 3072 + txt_in_dim: 5120 + txt_dim: ${.vid_dim} + emb_dim: ${eval:'6 * ${.vid_dim}'} + heads: 24 + head_dim: 128 # llm-like + expand_ratio: 4 + norm: fusedrms + norm_eps: 1e-5 + ada: single + qk_bias: False + qk_rope: True + qk_norm: fusedrms + patch_size: [1, 2, 2] + num_layers: 36 # llm-like + shared_mlp: False + shared_qkv: False + mlp_type: normal + block_type: ${eval:'${.num_layers} * ["mmdit_sr"]'} # space-full + window: ${eval:'${.num_layers} * [(4,3,3)]'} # space-full + window_method: ${eval:'${.num_layers} // 2 * ["720pwin_by_size_bysize","720pswin_by_size_bysize"]'} # space-full + compile: False + gradient_checkpoint: True + fsdp: + sharding_strategy: _HYBRID_SHARD_ZERO2 + +ema: + decay: 0.9998 + +vae: + model: + __object__: + path: + - "SeedVR2_VideoUpscaler.src.models.video_vae_v3.modules.attn_video_vae" + - "SeedVR2_VideoUpscaler.src.models.video_vae_v3.modules.attn_video_vae" + - "modules.seedvr.src.models.video_vae_v3.modules.attn_video_vae" + name: "VideoAutoencoderKLWrapper" + args: "as_params" + freeze_encoder: False + # gradient_checkpoint: True + slicing: + split_size: 4 + memory_device: same + memory_limit: + conv_max_mem: 0.5 + norm_max_mem: 0.5 + checkpoint: ema_vae_fp16.safetensors + scaling_factor: 0.9152 + compile: False + grouping: False + dtype: float16 + +diffusion: + schedule: + type: lerp + T: 1000.0 + sampler: + type: euler + prediction_type: v_lerp + timesteps: + training: + type: logitnormal + loc: 0.0 + scale: 1.0 + sampling: + type: uniform_trailing + steps: 50 + transform: True + loss: + type: v_lerp + cfg: + scale: 7.5 + rescale: 0 + +condition: + i2v: 0.0 + v2v: 0.0 + sr: 1.0 + noise_scale: 0.25 diff --git a/modules/seedvr/rotary_embedding.py b/modules/seedvr/rotary_embedding.py new file mode 100644 index 000000000..021ae9e52 --- /dev/null +++ b/modules/seedvr/rotary_embedding.py @@ -0,0 +1,346 @@ +from __future__ import annotations +from typing import Literal +from math import pi +import torch +from torch.amp import autocast +from torch.nn import Module +from torch import nn, einsum, broadcast_tensors, is_tensor, Tensor +from einops import rearrange, repeat + +# helper functions + +def exists(val): + return val is not None + +def default(val, d): + return val if exists(val) else d + +# broadcat, as tortoise-tts was using it + +def broadcat(tensors, dim = -1): + broadcasted_tensors = broadcast_tensors(*tensors) + return torch.cat(broadcasted_tensors, dim = dim) + +def slice_at_dim(t, dim_slice: slice, *, dim): + dim += (t.ndim if dim < 0 else 0) + colons = [slice(None)] * t.ndim + colons[dim] = dim_slice + return t[tuple(colons)] + +# rotary embedding helper functions + +def rotate_half(x): + x = rearrange(x, '... (d r) -> ... d r', r = 2) + x1, x2 = x.unbind(dim = -1) + x = torch.stack((-x2, x1), dim = -1) + return rearrange(x, '... d r -> ... (d r)') + +@autocast('cuda', enabled = False) +def apply_rotary_emb( + freqs, + t, + start_index = 0, + scale = 1., + seq_dim = -2, + freqs_seq_dim = None +): + dtype = t.dtype + + if not exists(freqs_seq_dim): + if freqs.ndim == 2 or t.ndim == 3: + freqs_seq_dim = 0 + + if t.ndim == 3 or exists(freqs_seq_dim): + seq_len = t.shape[seq_dim] + freqs = slice_at_dim(freqs, slice(-seq_len, None), dim = freqs_seq_dim) + + rot_dim = freqs.shape[-1] + end_index = start_index + rot_dim + + assert rot_dim <= t.shape[-1], f'feature dimension {t.shape[-1]} is not of sufficient size to rotate in all the positions {rot_dim}' + + # Split t into three parts: left, middle (to be transformed), and right + t_left = t[..., :start_index] + t_middle = t[..., start_index:end_index] + t_right = t[..., end_index:] + + # Apply rotary embeddings without modifying t in place + t_transformed = (t_middle * freqs.cos() * scale) + (rotate_half(t_middle) * freqs.sin() * scale) + out = torch.cat((t_left, t_transformed, t_right), dim=-1) + + return out.type(dtype) + +# learned rotation helpers + +def apply_learned_rotations(rotations, t, start_index = 0, freq_ranges = None): + if exists(freq_ranges): + rotations = einsum('..., f -> ... f', rotations, freq_ranges) + rotations = rearrange(rotations, '... r f -> ... (r f)') + + rotations = repeat(rotations, '... n -> ... (n r)', r = 2) + return apply_rotary_emb(rotations, t, start_index = start_index) + +# classes + +class RotaryEmbedding(Module): + def __init__( + self, + dim, + custom_freqs: Tensor | None = None, + freqs_for: Literal['lang', 'pixel', 'constant'] = 'lang', + theta = 10000, + max_freq = 10, + num_freqs = 1, + learned_freq = False, + use_xpos = False, + xpos_scale_base = 512, + interpolate_factor = 1., + theta_rescale_factor = 1., + seq_before_head_dim = False, + cache_if_possible = True, + cache_max_seq_len = 8192 + ): + super().__init__() + # proposed by reddit user bloc97, to rescale rotary embeddings to longer sequence length without fine-tuning + # has some connection to NTK literature + # https://www.reddit.com/r/LocalLLaMA/comments/14lz7j5/ntkaware_scaled_rope_allows_llama_models_to_have/ + + theta *= theta_rescale_factor ** (dim / (dim - 2)) + + self.freqs_for = freqs_for + + if exists(custom_freqs): + freqs = custom_freqs + elif freqs_for == 'lang': + freqs = 1. / (theta ** (torch.arange(0, dim, 2)[:(dim // 2)].float() / dim)) + elif freqs_for == 'pixel': + freqs = torch.linspace(1., max_freq / 2, dim // 2) * pi + elif freqs_for == 'constant': + freqs = torch.ones(num_freqs).float() + + self.cache_if_possible = cache_if_possible + self.cache_max_seq_len = cache_max_seq_len + + self.register_buffer('cached_freqs', torch.zeros(cache_max_seq_len, dim), persistent = False) + self.cached_freqs_seq_len = 0 + + self.freqs = nn.Parameter(freqs, requires_grad = learned_freq) # pylint: disable=possibly-used-before-assignment + + self.learned_freq = learned_freq + + # dummy for device + + self.register_buffer('dummy', torch.tensor(0), persistent = False) + + # default sequence dimension + + self.seq_before_head_dim = seq_before_head_dim + self.default_seq_dim = -3 if seq_before_head_dim else -2 + + # interpolation factors + + assert interpolate_factor >= 1. + self.interpolate_factor = interpolate_factor + + # xpos + + self.use_xpos = use_xpos + + if not use_xpos: + return + + scale = (torch.arange(0, dim, 2) + 0.4 * dim) / (1.4 * dim) + self.scale_base = xpos_scale_base + + self.register_buffer('scale', scale, persistent = False) + self.register_buffer('cached_scales', torch.zeros(cache_max_seq_len, dim), persistent = False) + self.cached_scales_seq_len = 0 + + # add apply_rotary_emb as static method + + self.apply_rotary_emb = staticmethod(apply_rotary_emb) + + @property + def device(self): + return self.dummy.device + + def get_seq_pos(self, seq_len, device = None, dtype = None, offset = 0): + device = default(device, self.device) + dtype = default(dtype, self.cached_freqs.dtype) + + return (torch.arange(seq_len, device = device, dtype = dtype) + offset) / self.interpolate_factor + + def rotate_queries_or_keys(self, t, seq_dim = None, offset = 0, scale = None): + seq_dim = default(seq_dim, self.default_seq_dim) + + assert not self.use_xpos or exists(scale), 'you must use `.rotate_queries_and_keys` method instead and pass in both queries and keys, for length extrapolatable rotary embeddings' + + device, dtype, seq_len = t.device, t.dtype, t.shape[seq_dim] + + seq = self.get_seq_pos(seq_len, device = device, dtype = dtype, offset = offset) + + freqs = self.forward(seq, seq_len = seq_len, offset = offset) + + if seq_dim == -3: + freqs = rearrange(freqs, 'n d -> n 1 d') + + return apply_rotary_emb(freqs, t, scale = default(scale, 1.), seq_dim = seq_dim) + + def rotate_queries_with_cached_keys(self, q, k, seq_dim = None, offset = 0): + dtype, device, seq_dim = q.dtype, q.device, default(seq_dim, self.default_seq_dim) + + q_len, k_len = q.shape[seq_dim], k.shape[seq_dim] + assert q_len <= k_len + + q_scale = k_scale = 1. + + if self.use_xpos: + seq = self.get_seq_pos(k_len, dtype = dtype, device = device) + + q_scale = self.get_scale(seq[-q_len:]).type(dtype) + k_scale = self.get_scale(seq).type(dtype) + + rotated_q = self.rotate_queries_or_keys(q, seq_dim = seq_dim, scale = q_scale, offset = k_len - q_len + offset) + rotated_k = self.rotate_queries_or_keys(k, seq_dim = seq_dim, scale = k_scale ** -1) + + rotated_q = rotated_q.type(q.dtype) + rotated_k = rotated_k.type(k.dtype) + + return rotated_q, rotated_k + + def rotate_queries_and_keys(self, q, k, seq_dim = None): + seq_dim = default(seq_dim, self.default_seq_dim) + + assert self.use_xpos + device, dtype, seq_len = q.device, q.dtype, q.shape[seq_dim] + + seq = self.get_seq_pos(seq_len, dtype = dtype, device = device) + + freqs = self.forward(seq, seq_len = seq_len) + scale = self.get_scale(seq, seq_len = seq_len).to(dtype) + + if seq_dim == -3: + freqs = rearrange(freqs, 'n d -> n 1 d') + scale = rearrange(scale, 'n d -> n 1 d') + + rotated_q = apply_rotary_emb(freqs, q, scale = scale, seq_dim = seq_dim) + rotated_k = apply_rotary_emb(freqs, k, scale = scale ** -1, seq_dim = seq_dim) + + rotated_q = rotated_q.type(q.dtype) + rotated_k = rotated_k.type(k.dtype) + + return rotated_q, rotated_k + + def get_scale( + self, + t: Tensor, + seq_len: int | None = None, + offset = 0 + ): + assert self.use_xpos + + should_cache = ( + self.cache_if_possible and + exists(seq_len) and + (offset + seq_len) <= self.cache_max_seq_len + ) + + if ( + should_cache and \ + exists(self.cached_scales) and \ + (seq_len + offset) <= self.cached_scales_seq_len + ): + return self.cached_scales[offset:(offset + seq_len)] + + scale = 1. + if self.use_xpos: + power = (t - len(t) // 2) / self.scale_base + scale = self.scale ** rearrange(power, 'n -> n 1') + scale = repeat(scale, 'n d -> n (d r)', r = 2) + + if should_cache and offset == 0: + self.cached_scales[:seq_len] = scale.detach() + self.cached_scales_seq_len = seq_len + + return scale + + def get_axial_freqs( + self, + *dims, + offsets: ( + tuple[int | float, ...] | + Tensor | + None + ) = None + ): + Colon = slice(None) + all_freqs = [] + + # handle offset + + if exists(offsets): + if not is_tensor(offsets): + offsets = torch.tensor(offsets) + + assert len(offsets) == len(dims) + + # get frequencies for each axis + + for ind, dim in enumerate(dims): + + offset = 0 + if exists(offsets): + offset = offsets[ind] + + if self.freqs_for == 'pixel': + pos = torch.linspace(-1, 1, steps = dim, device = self.device) + else: + pos = torch.arange(dim, device = self.device) + + pos = pos + offset + + freqs = self.forward(pos, seq_len = dim) + + all_axis = [None] * len(dims) + all_axis[ind] = Colon + + new_axis_slice = (Ellipsis, *all_axis, Colon) + all_freqs.append(freqs[new_axis_slice]) + + # concat all freqs + + all_freqs = broadcast_tensors(*all_freqs) + return torch.cat(all_freqs, dim = -1) + + @autocast('cuda', enabled = False) + def forward( + self, + t: Tensor, + seq_len: int | None = None, + offset = 0 + ): + should_cache = ( + self.cache_if_possible and + not self.learned_freq and + exists(seq_len) and + self.freqs_for != 'pixel' and + (offset + seq_len) <= self.cache_max_seq_len + ) + + if ( + should_cache and \ + exists(self.cached_freqs) and \ + (offset + seq_len) <= self.cached_freqs_seq_len + ): + return self.cached_freqs[offset:(offset + seq_len)].detach() + + freqs = self.freqs + + freqs = einsum('..., f -> ... f', t.type(freqs.dtype), freqs) + freqs = repeat(freqs, '... n -> ... (n r)', r = 2) + + if should_cache and offset == 0: + self.cached_freqs[:seq_len] = freqs.detach() + self.cached_freqs_seq_len = seq_len + + return freqs diff --git a/modules/seedvr/src/__init__.py b/modules/seedvr/src/__init__.py new file mode 100644 index 000000000..e2a119280 --- /dev/null +++ b/modules/seedvr/src/__init__.py @@ -0,0 +1,11 @@ +""" +# Core imports (always available) +import os +import sys + +# Add current directory to path for fallback imports +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +if parent_dir not in sys.path: + sys.path.insert(0, parent_dir) +""" diff --git a/modules/seedvr/src/common/__init__.py b/modules/seedvr/src/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/modules/seedvr/src/common/cache.py b/modules/seedvr/src/common/cache.py new file mode 100644 index 000000000..3566852ba --- /dev/null +++ b/modules/seedvr/src/common/cache.py @@ -0,0 +1,33 @@ +from typing import Callable + + +class Cache: + """Caching reusable args for faster inference""" + + def __init__(self, disable=False, prefix="", cache=None): + self.cache = cache if cache is not None else {} + self.disable = disable + self.prefix = prefix + + def __call__(self, key: str, fn: Callable): + if self.disable: + return fn() + + key = self.prefix + key + try: + result = self.cache[key] + except KeyError: + result = fn() + self.cache[key] = result + return result + + def namespace(self, namespace: str): + return Cache( + disable=self.disable, + prefix=self.prefix + namespace + ".", + cache=self.cache, + ) + + def get(self, key: str): + key = self.prefix + key + return self.cache[key] diff --git a/modules/seedvr/src/common/config.py b/modules/seedvr/src/common/config.py new file mode 100644 index 000000000..58c4e71f8 --- /dev/null +++ b/modules/seedvr/src/common/config.py @@ -0,0 +1,128 @@ +import importlib +from typing import Any, Callable, List, Union +from omegaconf import DictConfig, ListConfig, OmegaConf + + +try: + OmegaConf.register_new_resolver("eval", eval) +except Exception as e: + if "already registered" not in str(e): + raise + + +def load_config(path: str, argv: List[str] = None) -> Union[DictConfig, ListConfig]: + """ + Load a configuration. Will resolve inheritance. + """ + + config = OmegaConf.load(path) + if argv is not None: + config_argv = OmegaConf.from_dotlist(argv) + config = OmegaConf.merge(config, config_argv) + config = resolve_recursive(config, resolve_inheritance) + return config + + +def resolve_recursive( + config: Any, + resolver: Callable[[Union[DictConfig, ListConfig]], Union[DictConfig, ListConfig]], +) -> Any: + config = resolver(config) + if isinstance(config, DictConfig): + for k in config.keys(): + v = config.get(k) + if isinstance(v, (DictConfig, ListConfig)): + config[k] = resolve_recursive(v, resolver) + if isinstance(config, ListConfig): + for i in range(len(config)): + v = config.get(i) + if isinstance(v, (DictConfig, ListConfig)): + config[i] = resolve_recursive(v, resolver) + return config + + +def resolve_inheritance(config: Union[DictConfig, ListConfig]) -> Any: + """ + Recursively resolve inheritance if the config contains: + __inherit__: path/to/parent.yaml or a ListConfig of such paths. + """ + if isinstance(config, DictConfig): + inherit = config.pop("__inherit__", None) + + if inherit: + inherit_list = inherit if isinstance(inherit, ListConfig) else [inherit] + + parent_config = None + for parent_path in inherit_list: + assert isinstance(parent_path, str) + parent_config = ( + load_config(parent_path) + if parent_config is None + else OmegaConf.merge(parent_config, load_config(parent_path)) + ) + + if len(config.keys()) > 0: + config = OmegaConf.merge(parent_config, config) + else: + config = parent_config + return config + + +def import_item(path: Union[str, List[str]], name: str) -> Any: + """ + Import a python item with fallback support. + + Args: + path: Single path string or list of paths to try (fallback order) + name: Class/function name to import + + Returns: + Imported object + + Example: + import_item("path.to.file", "MyClass") -> MyClass + import_item(["path1.to.file", "path2.to.file"], "MyClass") -> MyClass (first working path) + """ + if isinstance(path, str): + # Single path - original behavior + return getattr(importlib.import_module(path), name) + + elif isinstance(path, (list, ListConfig)): + # Multiple paths - try each until one works + last_error = None + for single_path in path: + try: + return getattr(importlib.import_module(single_path), name) + except ImportError as e: + last_error = e + continue + + # If we get here, none of the paths worked + raise ImportError(f"Could not import '{name}' from any of the paths: {path}. Last error: {last_error}") + + else: + raise ValueError(f"Path must be string or list of strings, got: {type(path)}") + + +def create_object(config: DictConfig) -> Any: + """ + Create an object from config. + The config is expected to contains the following: + __object__: + path: path.to.module + name: MyClass + args: as_config | as_params (default to as_config) + """ + + item = import_item( + path=config.__object__.path, + name=config.__object__.name, + ) + args = config.__object__.get("args", "as_config") + if args == "as_config": + return item(config) + if args == "as_params": + config = OmegaConf.to_object(config) + config.pop("__object__") + return item(**config) + raise NotImplementedError(f"Unknown args type: {args}") diff --git a/modules/seedvr/src/common/decorators.py b/modules/seedvr/src/common/decorators.py new file mode 100644 index 000000000..cf504b3bf --- /dev/null +++ b/modules/seedvr/src/common/decorators.py @@ -0,0 +1,126 @@ +import functools +import threading +from typing import Callable +import torch +from .distributed import barrier_if_distributed, get_global_rank, get_local_rank +from .logger import get_logger + + +logger = get_logger(__name__) + + +def log_on_entry(func: Callable) -> Callable: + """ + Functions with this decorator will log the function name at entry. + When using multiple decorators, this must be applied innermost to properly capture the name. + """ + + def log_on_entry_wrapper(*args, **kwargs): + logger.info(f"Entering {func.__name__}") + return func(*args, **kwargs) + + return log_on_entry_wrapper + + +def barrier_on_entry(func: Callable) -> Callable: + """ + Functions with this decorator will start executing when all ranks are ready to enter. + """ + + def barrier_on_entry_wrapper(*args, **kwargs): + barrier_if_distributed() + return func(*args, **kwargs) + + return barrier_on_entry_wrapper + + +def _conditional_execute_wrapper_factory(execute: bool, func: Callable) -> Callable: + """ + Helper function for local_rank_zero_only and global_rank_zero_only. + """ + + def conditional_execute_wrapper(*args, **kwargs): + # Only execute if needed. + result = func(*args, **kwargs) if execute else None + # All GPUs must wait. + barrier_if_distributed() + # Return results. + return result + + return conditional_execute_wrapper + + +def _asserted_wrapper_factory(condition: bool, func: Callable, err_msg: str = "") -> Callable: + """ + Helper function for some functions with special constraints, + especially functions called by other global_rank_zero_only / local_rank_zero_only ones, + in case they are wrongly invoked in other scenarios. + """ + + def asserted_execute_wrapper(*args, **kwargs): + assert condition, err_msg + result = func(*args, **kwargs) + return result + + return asserted_execute_wrapper + + +def local_rank_zero_only(func: Callable) -> Callable: + """ + Functions with this decorator will only execute on local rank zero. + """ + return _conditional_execute_wrapper_factory(get_local_rank() == 0, func) + + +def global_rank_zero_only(func: Callable) -> Callable: + """ + Functions with this decorator will only execute on global rank zero. + """ + return _conditional_execute_wrapper_factory(get_global_rank() == 0, func) + + +def assert_only_global_rank_zero(func: Callable) -> Callable: + """ + Functions with this decorator are only accessible to processes with global rank zero. + """ + return _asserted_wrapper_factory( + get_global_rank() == 0, func, err_msg="Not accessible to processes with global_rank != 0" + ) + + +def assert_only_local_rank_zero(func: Callable) -> Callable: + """ + Functions with this decorator are only accessible to processes with local rank zero. + """ + return _asserted_wrapper_factory( + get_local_rank() == 0, func, err_msg="Not accessible to processes with local_rank != 0" + ) + + +def new_thread(func: Callable) -> Callable: + """ + Functions with this decorator will run in a new thread. + The function will return the thread, which can be joined to wait for completion. + """ + + def new_thread_wrapper(*args, **kwargs): + thread = threading.Thread(target=func, args=args, kwargs=kwargs) + thread.start() + return thread + + return new_thread_wrapper + + +def log_runtime(func: Callable) -> Callable: + """ + Functions with this decorator will logging the runtime. + """ + + @functools.wraps(func) + def wrapped(*args, **kwargs): + torch.distributed.barrier() + result = func(*args, **kwargs) + torch.distributed.barrier() + return result + + return wrapped diff --git a/modules/seedvr/src/common/diffusion/__init__.py b/modules/seedvr/src/common/diffusion/__init__.py new file mode 100644 index 000000000..034e36ef7 --- /dev/null +++ b/modules/seedvr/src/common/diffusion/__init__.py @@ -0,0 +1,56 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Diffusion package. +""" + +from .config import ( + create_sampler_from_config, + create_sampling_timesteps_from_config, + create_schedule_from_config, +) +from .samplers.base import Sampler +from .samplers.euler import EulerSampler +from .schedules.base import Schedule +from .schedules.lerp import LinearInterpolationSchedule +from .timesteps.base import SamplingTimesteps, Timesteps +from .timesteps.sampling.trailing import UniformTrailingSamplingTimesteps +from .types import PredictionType, SamplingDirection +from .utils import classifier_free_guidance, classifier_free_guidance_dispatcher, expand_dims + +__all__ = [ + # Configs + "create_sampler_from_config", + "create_sampling_timesteps_from_config", + "create_schedule_from_config", + # Schedules + "Schedule", + "DiscreteVariancePreservingSchedule", + "LinearInterpolationSchedule", + # Samplers + "Sampler", + "EulerSampler", + # Timesteps + "Timesteps", + "SamplingTimesteps", + # Types + "PredictionType", + "SamplingDirection", + "UniformTrailingSamplingTimesteps", + # Utils + "classifier_free_guidance", + "classifier_free_guidance_dispatcher", + "expand_dims", +] diff --git a/modules/seedvr/src/common/diffusion/config.py b/modules/seedvr/src/common/diffusion/config.py new file mode 100644 index 000000000..0a7183584 --- /dev/null +++ b/modules/seedvr/src/common/diffusion/config.py @@ -0,0 +1,71 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Utility functions for creating schedules and samplers from config. +""" + +import torch +from omegaconf import DictConfig + +from .samplers.base import Sampler +from .samplers.euler import EulerSampler +from .schedules.base import Schedule +from .schedules.lerp import LinearInterpolationSchedule +from .timesteps.base import SamplingTimesteps +from .timesteps.sampling.trailing import UniformTrailingSamplingTimesteps + + +def create_schedule_from_config( + config: DictConfig, +) -> Schedule: + """ + Create a schedule from configuration. + """ + if config.type == "lerp": + return LinearInterpolationSchedule(T=config.get("T", 1.0)) + + raise NotImplementedError + + +def create_sampler_from_config( + config: DictConfig, + schedule: Schedule, + timesteps: SamplingTimesteps, +) -> Sampler: + """ + Create a sampler from configuration. + """ + if config.type == "euler": + return EulerSampler( + schedule=schedule, + timesteps=timesteps, + prediction_type=config.prediction_type, + ) + raise NotImplementedError + + +def create_sampling_timesteps_from_config( + config: DictConfig, + schedule: Schedule, + device: torch.device, +) -> SamplingTimesteps: + if config.type == "uniform_trailing": + return UniformTrailingSamplingTimesteps( + T=schedule.T, + steps=config.steps, + shift=config.get("shift", 1.0), + device=device, + ) + raise NotImplementedError diff --git a/modules/seedvr/src/common/diffusion/samplers/base.py b/modules/seedvr/src/common/diffusion/samplers/base.py new file mode 100644 index 000000000..8e65f1989 --- /dev/null +++ b/modules/seedvr/src/common/diffusion/samplers/base.py @@ -0,0 +1,108 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Sampler base class. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Callable +import torch +from tqdm import tqdm + +from ..schedules.base import Schedule +from ..timesteps.base import SamplingTimesteps +from ..types import PredictionType, SamplingDirection +from ..utils import assert_schedule_timesteps_compatible + + +@dataclass +class SamplerModelArgs: + x_t: torch.Tensor + t: torch.Tensor + i: int + + +class Sampler(ABC): + """ + Samplers are ODE/SDE solvers. + """ + + def __init__( + self, + schedule: Schedule, + timesteps: SamplingTimesteps, + prediction_type: PredictionType, + return_endpoint: bool = True, + ): + assert_schedule_timesteps_compatible( + schedule=schedule, + timesteps=timesteps, + ) + self.schedule = schedule + self.timesteps = timesteps + self.prediction_type = prediction_type + self.return_endpoint = return_endpoint + + @abstractmethod + def sample( + self, + x: torch.Tensor, + f: Callable[[SamplerModelArgs], torch.Tensor], + ) -> torch.Tensor: + """ + Generate a new sample given the the intial sample x and score function f. + """ + + def get_next_timestep( + self, + t: torch.Tensor, + ) -> torch.Tensor: + """ + Get the next sample timestep. + Support multiple different timesteps t in a batch. + If no more steps, return out of bound value -1 or T+1. + """ + T = self.timesteps.T + steps = len(self.timesteps) + curr_idx = self.timesteps.index(t) + next_idx = curr_idx + 1 + bound = -1 if self.timesteps.direction == SamplingDirection.backward else T + 1 + + s = self.timesteps[next_idx.clamp_max(steps - 1)] + s = s.where(next_idx < steps, bound) + return s + + def get_endpoint( + self, + pred: torch.Tensor, + x_t: torch.Tensor, + t: torch.Tensor, + ) -> torch.Tensor: + """ + Get to the endpoint of the probability flow. + """ + x_0, x_T = self.schedule.convert_from_pred(pred, self.prediction_type, x_t, t) + return x_0 if self.timesteps.direction == SamplingDirection.backward else x_T + + def get_progress_bar(self): + """ + Get progress bar for sampling. + """ + return tqdm( + iterable=range(len(self.timesteps) - (0 if self.return_endpoint else 1)), + dynamic_ncols=True, + desc=self.__class__.__name__, + ) diff --git a/modules/seedvr/src/common/diffusion/samplers/euler.py b/modules/seedvr/src/common/diffusion/samplers/euler.py new file mode 100644 index 000000000..185669acb --- /dev/null +++ b/modules/seedvr/src/common/diffusion/samplers/euler.py @@ -0,0 +1,107 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + + +""" +Euler ODE solver. +""" + +from typing import Callable +import torch +from einops import rearrange +from torch.nn import functional as F + +#from ....models.dit_v2 import na + +from ..types import PredictionType +from ..utils import expand_dims +from .base import Sampler, SamplerModelArgs + + +class EulerSampler(Sampler): + """ + The Euler method is the simplest ODE solver. + + """ + + def sample( + self, + x: torch.Tensor, + f: Callable[[SamplerModelArgs], torch.Tensor], + ) -> torch.Tensor: + timesteps = self.timesteps.timesteps + #progress = self.get_progress_bar() + i = 0 + + # Optimisations VRAM + original_dtype = x.dtype + device = x.device + + for t, s in zip(timesteps[:-1], timesteps[1:]): + # Appel du modèle avec monitoring + pred = f(SamplerModelArgs(x, t, i)) + + # Étape suivante + x = self.step_to(pred, x, t, s) + + # Nettoyer les tenseurs temporaires + del pred + + i += 1 + #progress.update() + + if self.return_endpoint: + t = timesteps[-1] + pred = f(SamplerModelArgs(x, t, i)) + x = self.get_endpoint(pred, x, t) + del pred + #progress.update() + + # Restaurer le dtype original si nécessaire + if original_dtype != torch.float16: + x = x.to(original_dtype) + + return x + + def step( + self, + pred: torch.Tensor, + x_t: torch.Tensor, + t: torch.Tensor, + ) -> torch.Tensor: + """ + Step to the next timestep. + """ + return self.step_to(pred, x_t, t, self.get_next_timestep(t)) + + def step_to( + self, + pred: torch.Tensor, + x_t: torch.Tensor, + t: torch.Tensor, + s: torch.Tensor, + ) -> torch.Tensor: + """ + Steps from x_t at timestep t to x_s at timestep s. Returns x_s. + """ + t = expand_dims(t, x_t.ndim) + s = expand_dims(s, x_t.ndim) + T = self.schedule.T + # Step from x_t to x_s. + pred_x_0, pred_x_T = self.schedule.convert_from_pred(pred, self.prediction_type, x_t, t) + pred_x_s = self.schedule.forward(pred_x_0, pred_x_T, s.clamp(0, T)) + # Clamp x_s to x_0 and x_T if s is out of bound. + pred_x_s = pred_x_s.where(s >= 0, pred_x_0) + pred_x_s = pred_x_s.where(s <= T, pred_x_T) + return pred_x_s diff --git a/modules/seedvr/src/common/diffusion/schedules/base.py b/modules/seedvr/src/common/diffusion/schedules/base.py new file mode 100644 index 000000000..bcf6c6b64 --- /dev/null +++ b/modules/seedvr/src/common/diffusion/schedules/base.py @@ -0,0 +1,131 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Schedule base class. +""" + +from abc import ABC, abstractmethod, abstractproperty +from typing import Tuple, Union +import torch + +from ..types import PredictionType +from ..utils import expand_dims + + +class Schedule(ABC): + """ + Diffusion schedules are uniquely defined by T, A, B: + + x_t = A(t) * x_0 + B(t) * x_T, where t in [0, T] + + Schedules can be continuous or discrete. + """ + + @abstractproperty + def T(self) -> Union[int, float]: + """ + Maximum timestep inclusive. + Schedule is continuous if float, discrete if int. + """ + + @abstractmethod + def A(self, t: torch.Tensor) -> torch.Tensor: + """ + Interpolation coefficient A. + Returns tensor with the same shape as t. + """ + + @abstractmethod + def B(self, t: torch.Tensor) -> torch.Tensor: + """ + Interpolation coefficient B. + Returns tensor with the same shape as t. + """ + + # ---------------------------------------------------- + + def snr(self, t: torch.Tensor) -> torch.Tensor: + """ + Signal to noise ratio. + Returns tensor with the same shape as t. + """ + return (self.A(t) ** 2) / (self.B(t) ** 2) + + def isnr(self, snr: torch.Tensor) -> torch.Tensor: + """ + Inverse signal to noise ratio. + Returns tensor with the same shape as snr. + Subclass may implement. + """ + raise NotImplementedError + + # ---------------------------------------------------- + + def is_continuous(self) -> bool: + """ + Whether the schedule is continuous. + """ + return isinstance(self.T, float) + + def forward(self, x_0: torch.Tensor, x_T: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + """ + Diffusion forward function. + """ + t = expand_dims(t, x_0.ndim) + return self.A(t) * x_0 + self.B(t) * x_T + + def convert_from_pred( + self, pred: torch.Tensor, pred_type: PredictionType, x_t: torch.Tensor, t: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Convert from prediction. Return predicted x_0 and x_T. + """ + t = expand_dims(t, x_t.ndim) + A_t = self.A(t) + B_t = self.B(t) + + if pred_type == PredictionType.x_T: + pred_x_T = pred + pred_x_0 = (x_t - B_t * pred_x_T) / A_t + elif pred_type == PredictionType.x_0: + pred_x_0 = pred + pred_x_T = (x_t - A_t * pred_x_0) / B_t + elif pred_type == PredictionType.v_cos: + pred_x_0 = A_t * x_t - B_t * pred + pred_x_T = A_t * pred + B_t * x_t + elif pred_type == PredictionType.v_lerp: + pred_x_0 = (x_t - B_t * pred) / (A_t + B_t) + pred_x_T = (x_t + A_t * pred) / (A_t + B_t) + else: + raise NotImplementedError + + return pred_x_0, pred_x_T + + def convert_to_pred( + self, x_0: torch.Tensor, x_T: torch.Tensor, t: torch.Tensor, pred_type: PredictionType + ) -> torch.FloatTensor: + """ + Convert to prediction target given x_0 and x_T. + """ + if pred_type == PredictionType.x_T: + return x_T + if pred_type == PredictionType.x_0: + return x_0 + if pred_type == PredictionType.v_cos: + t = expand_dims(t, x_0.ndim) + return self.A(t) * x_T - self.B(t) * x_0 + if pred_type == PredictionType.v_lerp: + return x_T - x_0 + raise NotImplementedError diff --git a/modules/seedvr/src/common/diffusion/schedules/lerp.py b/modules/seedvr/src/common/diffusion/schedules/lerp.py new file mode 100644 index 000000000..56b42bc17 --- /dev/null +++ b/modules/seedvr/src/common/diffusion/schedules/lerp.py @@ -0,0 +1,55 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Linear interpolation schedule (lerp). +""" + +from typing import Union +import torch + +from .base import Schedule + + +class LinearInterpolationSchedule(Schedule): + """ + Linear interpolation schedule (lerp) is proposed by flow matching and rectified flow. + It leads to straighter probability flow theoretically. It is also used by Stable Diffusion 3. + + + + x_t = (1 - t) * x_0 + t * x_T + + Can be either continuous or discrete. + """ + + def __init__(self, T: Union[int, float] = 1.0): + self._T = T + + @property + def T(self) -> Union[int, float]: + return self._T + + def A(self, t: torch.Tensor) -> torch.Tensor: + return 1 - (t / self.T) + + def B(self, t: torch.Tensor) -> torch.Tensor: + return t / self.T + + # ---------------------------------------------------- + + def isnr(self, snr: torch.Tensor) -> torch.Tensor: + t = self.T / (1 + snr**0.5) + t = t if self.is_continuous() else t.round().int() + return t diff --git a/modules/seedvr/src/common/diffusion/timesteps/base.py b/modules/seedvr/src/common/diffusion/timesteps/base.py new file mode 100644 index 000000000..d1a598103 --- /dev/null +++ b/modules/seedvr/src/common/diffusion/timesteps/base.py @@ -0,0 +1,72 @@ +from abc import ABC, abstractmethod +from typing import Sequence, Union +import torch + +from ..types import SamplingDirection + + +class Timesteps(ABC): + """ + Timesteps base class. + """ + + def __init__(self, T: Union[int, float]): + assert T > 0 + self._T = T + + @property + def T(self) -> Union[int, float]: + """ + Maximum timestep inclusive. + int if discrete, float if continuous. + """ + return self._T + + def is_continuous(self) -> bool: + """ + Whether the schedule is continuous. + """ + return isinstance(self.T, float) + + +class SamplingTimesteps(Timesteps): + """ + Sampling timesteps. + It defines the discretization of sampling steps. + """ + + def __init__( + self, + T: Union[int, float], + timesteps: torch.Tensor, + direction: SamplingDirection, + ): + assert timesteps.ndim == 1 + super().__init__(T) + self.timesteps = timesteps + self.direction = direction + + def __len__(self) -> int: + """ + Number of sampling steps. + """ + return len(self.timesteps) + + def __getitem__(self, idx: Union[int, torch.IntTensor]) -> torch.Tensor: + """ + The timestep at the sampling step. + Returns a scalar tensor if idx is int, + or tensor of the same size if idx is a tensor. + """ + return self.timesteps[idx] + + def index(self, t: torch.Tensor) -> torch.Tensor: + """ + Find index by t. + Return index of the same shape as t. + Index is -1 if t not found in timesteps. + """ + i, j = t.reshape(-1, 1).eq(self.timesteps).nonzero(as_tuple=True) + idx = torch.full_like(t, fill_value=-1, dtype=torch.int) + idx.view(-1)[i] = j.int() + return idx diff --git a/modules/seedvr/src/common/diffusion/timesteps/sampling/trailing.py b/modules/seedvr/src/common/diffusion/timesteps/sampling/trailing.py new file mode 100644 index 000000000..248d986ae --- /dev/null +++ b/modules/seedvr/src/common/diffusion/timesteps/sampling/trailing.py @@ -0,0 +1,49 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +import torch + +from ...types import SamplingDirection +from ..base import SamplingTimesteps + + +class UniformTrailingSamplingTimesteps(SamplingTimesteps): + """ + Uniform trailing sampling timesteps. + Defined in (https://arxiv.org/abs/2305.08891) + + Shift is proposed in SD3 for RF schedule. + Defined in (https://arxiv.org/pdf/2403.03206) eq.23 + """ + + def __init__( + self, + T: int, + steps: int, + shift: float = 1.0, + device: torch.device = "cpu", + ): + # Create trailing timesteps. + timesteps = torch.arange(1.0, 0.0, -1.0 / steps, device=device) + + # Shift timesteps. + timesteps = shift * timesteps / (1 + (shift - 1) * timesteps) + + # Scale to T range. + if isinstance(T, float): + timesteps = timesteps * T + else: + timesteps = timesteps.mul(T + 1).sub(1).round().int() + + super().__init__(T=T, timesteps=timesteps, direction=SamplingDirection.backward) diff --git a/modules/seedvr/src/common/diffusion/types.py b/modules/seedvr/src/common/diffusion/types.py new file mode 100644 index 000000000..076295f2b --- /dev/null +++ b/modules/seedvr/src/common/diffusion/types.py @@ -0,0 +1,59 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Type definitions. +""" + +from enum import Enum + + +class PredictionType(str, Enum): + """ + x_0: + Predict data sample. + x_T: + Predict noise sample. + Proposed by DDPM (https://arxiv.org/abs/2006.11239) + Proved problematic by zsnr paper (https://arxiv.org/abs/2305.08891) + v_cos: + Predict velocity dx/dt based on the cosine schedule (A_t * x_T - B_t * x_0). + Proposed by progressive distillation (https://arxiv.org/abs/2202.00512) + v_lerp: + Predict velocity dx/dt based on the lerp schedule (x_T - x_0). + Proposed by rectified flow (https://arxiv.org/abs/2209.03003) + """ + + x_0 = "x_0" + x_T = "x_T" + v_cos = "v_cos" + v_lerp = "v_lerp" + + +class SamplingDirection(str, Enum): + """ + backward: Sample from x_T to x_0 for data generation. + forward: Sample from x_0 to x_T for noise inversion. + """ + + backward = "backward" + forward = "forward" + + @staticmethod + def reverse(direction): + if direction == SamplingDirection.backward: + return SamplingDirection.forward + if direction == SamplingDirection.forward: + return SamplingDirection.backward + raise NotImplementedError diff --git a/modules/seedvr/src/common/diffusion/utils.py b/modules/seedvr/src/common/diffusion/utils.py new file mode 100644 index 000000000..69d4aec34 --- /dev/null +++ b/modules/seedvr/src/common/diffusion/utils.py @@ -0,0 +1,84 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Utility functions. +""" + +from typing import Callable +import torch + + +def expand_dims(tensor: torch.Tensor, ndim: int): + """ + Expand tensor to target ndim. New dims are added to the right. + For example, if the tensor shape was (8,), target ndim is 4, return (8, 1, 1, 1). + """ + shape = tensor.shape + (1,) * (ndim - tensor.ndim) + return tensor.reshape(shape) + + +def assert_schedule_timesteps_compatible(schedule, timesteps): + """ + Check if schedule and timesteps are compatible. + """ + if schedule.T != timesteps.T: + raise ValueError("Schedule and timesteps must have the same T.") + if schedule.is_continuous() != timesteps.is_continuous(): + raise ValueError("Schedule and timesteps must have the same continuity.") + + +def classifier_free_guidance( + pos: torch.Tensor, + neg: torch.Tensor, + scale: float, + rescale: float = 0.0, +): + """ + Apply classifier-free guidance. + """ + # Classifier-free guidance (https://arxiv.org/abs/2207.12598) + cfg = neg + scale * (pos - neg) + + # Classifier-free guidance rescale (https://arxiv.org/pdf/2305.08891.pdf) + if rescale != 0.0: + pos_std = pos.std(dim=list(range(1, pos.ndim)), keepdim=True) + cfg_std = cfg.std(dim=list(range(1, cfg.ndim)), keepdim=True) + factor = pos_std / cfg_std + factor = rescale * factor + (1 - rescale) + cfg *= factor + + return cfg + + +def classifier_free_guidance_dispatcher( + pos: Callable, + neg: Callable, + scale: float, + rescale: float = 0.0, +): + """ + Optionally execute models depending on classifer-free guidance scale. + """ + # If scale is 1, no need to execute neg model. + if scale == 1.0: + return pos() + + # Otherwise, execute both pos nad neg models and apply cfg. + return classifier_free_guidance( + pos=pos(), + neg=neg(), + scale=scale, + rescale=rescale, + ) diff --git a/modules/seedvr/src/common/distributed/__init__.py b/modules/seedvr/src/common/distributed/__init__.py new file mode 100644 index 000000000..9f53a78eb --- /dev/null +++ b/modules/seedvr/src/common/distributed/__init__.py @@ -0,0 +1,35 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Distributed package. +""" + +from .basic import ( + barrier_if_distributed, + convert_to_ddp, + get_device, + get_global_rank, + get_local_rank, + get_world_size, +) + +__all__ = [ + "barrier_if_distributed", + "convert_to_ddp", + "get_device", + "get_global_rank", + "get_local_rank", + "get_world_size", +] diff --git a/modules/seedvr/src/common/distributed/advanced.py b/modules/seedvr/src/common/distributed/advanced.py new file mode 100644 index 000000000..d2479bebf --- /dev/null +++ b/modules/seedvr/src/common/distributed/advanced.py @@ -0,0 +1,208 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Advanced distributed functions for sequence parallel. +""" + +from typing import Optional, List +import torch +import torch.distributed as dist +from torch.distributed.device_mesh import DeviceMesh, init_device_mesh +from torch.distributed.fsdp import ShardingStrategy + +from .basic import get_global_rank, get_world_size + + +_DATA_PARALLEL_GROUP = None +_SEQUENCE_PARALLEL_GROUP = None +_SEQUENCE_PARALLEL_CPU_GROUP = None +_MODEL_SHARD_CPU_INTER_GROUP = None +_MODEL_SHARD_CPU_INTRA_GROUP = None +_MODEL_SHARD_INTER_GROUP = None +_MODEL_SHARD_INTRA_GROUP = None +_SEQUENCE_PARALLEL_GLOBAL_RANKS = None + + +def get_data_parallel_group() -> Optional[dist.ProcessGroup]: + """ + Get data parallel process group. + """ + return _DATA_PARALLEL_GROUP + + +def get_sequence_parallel_group() -> Optional[dist.ProcessGroup]: + """ + Get sequence parallel process group. + """ + return _SEQUENCE_PARALLEL_GROUP + + +def get_sequence_parallel_cpu_group() -> Optional[dist.ProcessGroup]: + """ + Get sequence parallel CPU process group. + """ + return _SEQUENCE_PARALLEL_CPU_GROUP + + +def get_data_parallel_rank() -> int: + """ + Get data parallel rank. + """ + group = get_data_parallel_group() + return dist.get_rank(group) if group else get_global_rank() + + +def get_data_parallel_world_size() -> int: + """ + Get data parallel world size. + """ + group = get_data_parallel_group() + return dist.get_world_size(group) if group else get_world_size() + + +def get_sequence_parallel_rank() -> int: + """ + Get sequence parallel rank. + """ + group = get_sequence_parallel_group() + return dist.get_rank(group) if group else 0 + + +def get_sequence_parallel_world_size() -> int: + """ + Get sequence parallel world size. + """ + group = get_sequence_parallel_group() + return dist.get_world_size(group) if group else 1 + + +def get_model_shard_cpu_intra_group() -> Optional[dist.ProcessGroup]: + """ + Get the CPU intra process group of model sharding. + """ + return _MODEL_SHARD_CPU_INTRA_GROUP + + +def get_model_shard_cpu_inter_group() -> Optional[dist.ProcessGroup]: + """ + Get the CPU inter process group of model sharding. + """ + return _MODEL_SHARD_CPU_INTER_GROUP + + +def get_model_shard_intra_group() -> Optional[dist.ProcessGroup]: + """ + Get the GPU intra process group of model sharding. + """ + return _MODEL_SHARD_INTRA_GROUP + + +def get_model_shard_inter_group() -> Optional[dist.ProcessGroup]: + """ + Get the GPU inter process group of model sharding. + """ + return _MODEL_SHARD_INTER_GROUP + + +def init_sequence_parallel(sequence_parallel_size: int): + """ + Initialize sequence parallel. + """ + global _DATA_PARALLEL_GROUP + global _SEQUENCE_PARALLEL_GROUP + global _SEQUENCE_PARALLEL_CPU_GROUP + global _SEQUENCE_PARALLEL_GLOBAL_RANKS + assert dist.is_initialized() + world_size = dist.get_world_size() + rank = dist.get_rank() + data_parallel_size = world_size // sequence_parallel_size + for i in range(data_parallel_size): + start_rank = i * sequence_parallel_size + end_rank = (i + 1) * sequence_parallel_size + ranks = range(start_rank, end_rank) + group = dist.new_group(ranks) + cpu_group = dist.new_group(ranks, backend="gloo") + if rank in ranks: + _SEQUENCE_PARALLEL_GROUP = group + _SEQUENCE_PARALLEL_CPU_GROUP = cpu_group + _SEQUENCE_PARALLEL_GLOBAL_RANKS = list(ranks) + + +def init_model_shard_group( + *, + sharding_strategy: ShardingStrategy, + device_mesh: Optional[DeviceMesh] = None, +): + """ + Initialize process group of model sharding. + """ + global _MODEL_SHARD_INTER_GROUP + global _MODEL_SHARD_INTRA_GROUP + global _MODEL_SHARD_CPU_INTER_GROUP + global _MODEL_SHARD_CPU_INTRA_GROUP + assert dist.is_initialized() + world_size = dist.get_world_size() + if device_mesh is not None: + num_shards_per_group = device_mesh.shape[1] + elif sharding_strategy == ShardingStrategy.NO_SHARD: + num_shards_per_group = 1 + elif sharding_strategy in [ + ShardingStrategy.HYBRID_SHARD, + ShardingStrategy._HYBRID_SHARD_ZERO2, + ]: + num_shards_per_group = torch.cuda.device_count() + else: + num_shards_per_group = world_size + num_groups = world_size // num_shards_per_group + device_mesh = (num_groups, num_shards_per_group) + + gpu_mesh_2d = init_device_mesh("cuda", device_mesh, mesh_dim_names=("inter", "intra")) + cpu_mesh_2d = init_device_mesh("cpu", device_mesh, mesh_dim_names=("inter", "intra")) + + _MODEL_SHARD_INTER_GROUP = gpu_mesh_2d.get_group("inter") + _MODEL_SHARD_INTRA_GROUP = gpu_mesh_2d.get_group("intra") + _MODEL_SHARD_CPU_INTER_GROUP = cpu_mesh_2d.get_group("inter") + _MODEL_SHARD_CPU_INTRA_GROUP = cpu_mesh_2d.get_group("intra") + +def get_sequence_parallel_global_ranks() -> List[int]: + """ + Get all global ranks of the sequence parallel process group + that the caller rank belongs to. + """ + if _SEQUENCE_PARALLEL_GLOBAL_RANKS is None: + return [dist.get_rank()] + return _SEQUENCE_PARALLEL_GLOBAL_RANKS + + +def get_next_sequence_parallel_rank() -> int: + """ + Get the next global rank of the sequence parallel process group + that the caller rank belongs to. + """ + sp_global_ranks = get_sequence_parallel_global_ranks() + sp_rank = get_sequence_parallel_rank() + sp_size = get_sequence_parallel_world_size() + return sp_global_ranks[(sp_rank + 1) % sp_size] + + +def get_prev_sequence_parallel_rank() -> int: + """ + Get the previous global rank of the sequence parallel process group + that the caller rank belongs to. + """ + sp_global_ranks = get_sequence_parallel_global_ranks() + sp_rank = get_sequence_parallel_rank() + sp_size = get_sequence_parallel_world_size() + return sp_global_ranks[(sp_rank + sp_size - 1) % sp_size] diff --git a/modules/seedvr/src/common/distributed/basic.py b/modules/seedvr/src/common/distributed/basic.py new file mode 100644 index 000000000..d880009c9 --- /dev/null +++ b/modules/seedvr/src/common/distributed/basic.py @@ -0,0 +1,67 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Distributed basic functions. +""" + +import os +import torch + + +def get_global_rank() -> int: + """ + Get the global rank, the global index of the GPU. + """ + return int(os.environ.get("RANK", "0")) + + +def get_local_rank() -> int: + """ + Get the local rank, the local index of the GPU. + """ + return int(os.environ.get("LOCAL_RANK", "0")) + + +def get_world_size() -> int: + """ + Get the world size, the total amount of GPUs. + """ + return int(os.environ.get("WORLD_SIZE", "1")) + + +def get_device() -> torch.device: + """ + Get current rank device. + """ + return torch.device("cuda", get_local_rank()) + + +def barrier_if_distributed(*args, **kwargs): + """ + Synchronizes all processes if under distributed context. + """ + import torch.distributed as dist + if dist.is_initialized(): + return dist.barrier(*args, **kwargs) + + +def convert_to_ddp(module: torch.nn.Module, **kwargs): + from torch.nn.parallel import DistributedDataParallel + return DistributedDataParallel( + module=module, + device_ids=[get_local_rank()], + output_device=get_local_rank(), + **kwargs, + ) diff --git a/modules/seedvr/src/common/distributed/meta_init_utils.py b/modules/seedvr/src/common/distributed/meta_init_utils.py new file mode 100644 index 000000000..9a6570646 --- /dev/null +++ b/modules/seedvr/src/common/distributed/meta_init_utils.py @@ -0,0 +1,40 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +import torch +from torch import nn +from ....rotary_embedding import RotaryEmbedding + +__all__ = ["meta_non_persistent_buffer_init_fn"] + + +def meta_non_persistent_buffer_init_fn(module: nn.Module) -> nn.Module: + """ + Used for materializing `non-persistent tensor buffers` while model resuming. + + Since non-persistent tensor buffers are not saved in state_dict, + when initializing model with meta device, user should materialize those buffers manually. + + Currently, only `rope.dummy` is this special case. + """ + with torch.no_grad(): + for submodule in module.modules(): + if not isinstance(submodule, RotaryEmbedding): + continue + for buffer_name, buffer in submodule.named_buffers(recurse=False): + if buffer.is_meta and "dummy" in buffer_name: + materialized_buffer = torch.zeros_like(buffer, device="cpu") + setattr(submodule, buffer_name, materialized_buffer) + assert not any(b.is_meta for n, b in module.named_buffers()) + return module diff --git a/modules/seedvr/src/common/distributed/ops.py b/modules/seedvr/src/common/distributed/ops.py new file mode 100644 index 000000000..bba121b85 --- /dev/null +++ b/modules/seedvr/src/common/distributed/ops.py @@ -0,0 +1,493 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Distributed ops for supporting sequence parallel. +""" + +from collections import defaultdict +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +import torch +import torch.distributed as dist +from torch import Tensor + +from ..cache import Cache +from .advanced import ( + get_sequence_parallel_group, + get_sequence_parallel_rank, +) + +from .basic import get_device + +_SEQ_DATA_BUF = defaultdict(lambda: [None, None, None]) +_SEQ_DATA_META_SHAPES = defaultdict() +_SEQ_DATA_META_DTYPES = defaultdict() +_SEQ_DATA_ASYNC_COMMS = defaultdict(list) +_SYNC_BUFFER = defaultdict(dict) + + +def single_all_to_all( + local_input: Tensor, + scatter_dim: int, + gather_dim: int, + group: dist.ProcessGroup, + async_op: bool = False, +): + """ + A function to do all-to-all on a tensor + """ + seq_world_size = 1 + prev_scatter_dim = scatter_dim + if scatter_dim != 0: + local_input = local_input.transpose(0, scatter_dim) + if gather_dim == 0: + gather_dim = scatter_dim + scatter_dim = 0 + + inp_shape = list(local_input.shape) + inp_shape[scatter_dim] = inp_shape[scatter_dim] // seq_world_size + input_t = local_input.reshape( + [seq_world_size, inp_shape[scatter_dim]] + inp_shape[scatter_dim + 1 :] + ).contiguous() + output = torch.empty_like(input_t) + comm = dist.all_to_all_single(output, input_t, group=group, async_op=async_op) + if async_op: + # let user's code transpose & reshape + return output, comm, prev_scatter_dim + + # first dim is seq_world_size, so we can split it directly + output = torch.cat(output.split(1), dim=gather_dim + 1).squeeze(0) + if prev_scatter_dim: + output = output.transpose(0, prev_scatter_dim).contiguous() + return output + + +def _all_to_all( + local_input: Tensor, + scatter_dim: int, + gather_dim: int, + group: dist.ProcessGroup, +): + seq_world_size = 1 + input_list = [ + t.contiguous() for t in torch.tensor_split(local_input, seq_world_size, scatter_dim) + ] + output_list = [torch.empty_like(input_list[0]) for _ in range(seq_world_size)] + dist.all_to_all(output_list, input_list, group=group) + return torch.cat(output_list, dim=gather_dim).contiguous() + + +class SeqAllToAll(torch.autograd.Function): + @staticmethod + def forward( + ctx: Any, + group: dist.ProcessGroup, + local_input: Tensor, + scatter_dim: int, + gather_dim: int, + async_op: bool, + ) -> Tensor: + ctx.group = group + ctx.scatter_dim = scatter_dim + ctx.gather_dim = gather_dim + ctx.async_op = async_op + if async_op: + output, comm, prev_scatter_dim = single_all_to_all( + local_input, scatter_dim, gather_dim, group, async_op=async_op + ) + ctx.prev_scatter_dim = prev_scatter_dim + return output, comm + + return _all_to_all(local_input, scatter_dim, gather_dim, group) + + @staticmethod + def backward(ctx: Any, *grad_output: Tensor) -> Tuple[None, Tensor, None, None]: + if ctx.async_op: + input_t = torch.cat(grad_output[0].split(1), dim=ctx.gather_dim + 1).squeeze(0) + if ctx.prev_scatter_dim: + input_t = input_t.transpose(0, ctx.prev_scatter_dim) + else: + input_t = grad_output[0] + return ( + None, + _all_to_all(input_t, ctx.gather_dim, ctx.scatter_dim, ctx.group), + None, + None, + None, + ) + + +class Slice(torch.autograd.Function): + @staticmethod + def forward(ctx: Any, group: dist.ProcessGroup, local_input: Tensor, dim: int) -> Tensor: + ctx.group = group + ctx.rank = dist.get_rank(group) + seq_world_size = 1 + ctx.seq_world_size = seq_world_size + ctx.dim = dim + dim_size = local_input.shape[dim] + return local_input.split(dim_size // seq_world_size, dim=dim)[ctx.rank].contiguous() + + @staticmethod + def backward(ctx: Any, grad_output: Tensor) -> Tuple[None, Tensor, None]: + dim_size = list(grad_output.size()) + split_size = dim_size[0] + dim_size[0] = dim_size[0] * ctx.seq_world_size + output = torch.empty(dim_size, dtype=grad_output.dtype, device=torch.cuda.current_device()) + dist._all_gather_base(output, grad_output, group=ctx.group) + return (None, torch.cat(output.split(split_size), dim=ctx.dim), None) + + +class Gather(torch.autograd.Function): + @staticmethod + def forward( + ctx: Any, + group: dist.ProcessGroup, + local_input: Tensor, + dim: int, + grad_scale: Optional[bool] = False, + ) -> Tensor: + ctx.group = group + ctx.rank = dist.get_rank(group) + ctx.dim = dim + ctx.grad_scale = grad_scale + seq_world_size = 1 + ctx.seq_world_size = seq_world_size + dim_size = list(local_input.size()) + split_size = dim_size[0] + ctx.part_size = dim_size[dim] + dim_size[0] = dim_size[0] * seq_world_size + output = torch.empty(dim_size, dtype=local_input.dtype, device=torch.cuda.current_device()) + dist._all_gather_base(output, local_input.contiguous(), group=ctx.group) + return torch.cat(output.split(split_size), dim=dim) + + @staticmethod + def backward(ctx: Any, grad_output: Tensor) -> Tuple[None, Tensor]: + if ctx.grad_scale: + grad_output = grad_output * ctx.seq_world_size + return ( + None, + grad_output.split(ctx.part_size, dim=ctx.dim)[ctx.rank].contiguous(), + None, + None, + ) + + +def gather_seq_scatter_heads_qkv( + qkv_tensor: Tensor, + *, + seq_dim: int, + qkv_shape: Optional[Tensor] = None, + cache: Cache = Cache(disable=True), + restore_shape: bool = True, +): + """ + A func to sync splited qkv tensor + qkv_tensor: the tensor we want to do alltoall with. The last dim must + be the projection_idx, which we will split into 3 part. After + spliting, the gather idx will be projecttion_idx + 1 + seq_dim: gather_dim for all2all comm + restore_shape: if True, output will has the same shape length as input + """ + group = get_sequence_parallel_group() + if not group: + return qkv_tensor + world = 1 + orig_shape = qkv_tensor.shape + scatter_dim = qkv_tensor.dim() + bef_all2all_shape = list(orig_shape) + qkv_proj_dim = bef_all2all_shape[-1] + bef_all2all_shape = bef_all2all_shape[:-1] + [3, qkv_proj_dim // 3] + qkv_tensor = qkv_tensor.view(bef_all2all_shape) + qkv_tensor = SeqAllToAll.apply(group, qkv_tensor, scatter_dim, seq_dim, False) + if restore_shape: + out_shape = list(orig_shape) + out_shape[seq_dim] *= world + out_shape[-1] = qkv_proj_dim // world + qkv_tensor = qkv_tensor.view(out_shape) + + # remove padding + if qkv_shape is not None: + unpad_dim_size = cache( + "unpad_dim_size", lambda: torch.sum(torch.prod(qkv_shape, dim=-1)).item() + ) + if unpad_dim_size % world != 0: + padding_size = qkv_tensor.size(seq_dim) - unpad_dim_size + qkv_tensor = _unpad_tensor(qkv_tensor, seq_dim, padding_size) + return qkv_tensor + + +def slice_inputs(x: Tensor, dim: int, padding: bool = True): + """ + A func to slice the input sequence in sequence parallel + """ + group = get_sequence_parallel_group() + if group is None: + return x + sp_rank = get_sequence_parallel_rank() + sp_world = 1 + dim_size = x.shape[dim] + unit = (dim_size + sp_world - 1) // sp_world + if padding and dim_size % sp_world: + padding_size = sp_world - (dim_size % sp_world) + x = _pad_tensor(x, dim, padding_size) + slc = [slice(None)] * len(x.shape) + slc[dim] = slice(unit * sp_rank, unit * (sp_rank + 1)) + return x[slc] + + +def remove_seqeunce_parallel_padding(x: Tensor, dim: int, unpad_dim_size: int): + """ + A func to remove the padding part of the tensor based on its original shape + """ + group = get_sequence_parallel_group() + if group is None: + return x + sp_world = 1 + if unpad_dim_size % sp_world == 0: + return x + padding_size = sp_world - (unpad_dim_size % sp_world) + assert (padding_size + unpad_dim_size) % sp_world == 0 + return _unpad_tensor(x, dim=dim, padding_size=padding_size) + + +def gather_heads_scatter_seq(x: Tensor, head_dim: int, seq_dim: int) -> Tensor: + """ + A func to sync attention result with alltoall in sequence parallel + """ + group = get_sequence_parallel_group() + if not group: + return x + dim_size = x.size(seq_dim) + sp_world = 1 + if dim_size % sp_world != 0: + padding_size = sp_world - (dim_size % sp_world) + x = _pad_tensor(x, seq_dim, padding_size) + return SeqAllToAll.apply(group, x, seq_dim, head_dim, False) + + +def gather_seq_scatter_heads(x: Tensor, seq_dim: int, head_dim: int) -> Tensor: + """ + A func to sync embedding input with alltoall in sequence parallel + """ + group = get_sequence_parallel_group() + if not group: + return x + return SeqAllToAll.apply(group, x, head_dim, seq_dim, False) + + +def scatter_heads(x: Tensor, dim: int) -> Tensor: + """ + A func to split heads before attention in sequence parallel + """ + group = get_sequence_parallel_group() + if not group: + return x + return Slice.apply(group, x, dim) + + +def gather_heads(x: Tensor, dim: int, grad_scale: Optional[bool] = False) -> Tensor: + """ + A func to gather heads for the attention result in sequence parallel + """ + group = get_sequence_parallel_group() + if not group: + return x + return Gather.apply(group, x, dim, grad_scale) + + +def gather_outputs( + x: Tensor, + *, + gather_dim: int, + padding_dim: Optional[int] = None, + unpad_shape: Optional[Tensor] = None, + cache: Cache = Cache(disable=True), + scale_grad=True, +): + """ + A func to gather the outputs for the model result in sequence parallel + """ + group = get_sequence_parallel_group() + if not group: + return x + x = Gather.apply(group, x, gather_dim, scale_grad) + if padding_dim is not None: + unpad_dim_size = cache( + "unpad_dim_size", lambda: torch.sum(torch.prod(unpad_shape, dim=1)).item() + ) + x = remove_seqeunce_parallel_padding(x, padding_dim, unpad_dim_size) + return x + + +def _pad_tensor(x: Tensor, dim: int, padding_size: int): + shape = list(x.shape) + shape[dim] = padding_size + pad = torch.zeros(shape, dtype=x.dtype, device=x.device) + return torch.cat([x, pad], dim=dim) + + +def _unpad_tensor(x: Tensor, dim: int, padding_size): + slc = [slice(None)] * len(x.shape) + slc[dim] = slice(0, -padding_size) + return x[slc] + + +def _broadcast_data(data, shape, dtype, src, group, async_op): + comms = [] + if isinstance(data, (list, tuple)): + for i, sub_shape in enumerate(shape): + comms += _broadcast_data(data[i], sub_shape, dtype[i], src, group, async_op) + elif isinstance(data, dict): + for key, sub_data in data.items(): + comms += _broadcast_data(sub_data, shape[key], dtype[key], src, group, async_op) + elif isinstance(data, Tensor): + comms.append(dist.broadcast(data, src=src, group=group, async_op=async_op)) + return comms + + +def _traverse(data: Any, op: Callable) -> Union[None, List, Dict, Any]: + if isinstance(data, (list, tuple)): + return [_traverse(sub_data, op) for sub_data in data] + elif isinstance(data, dict): + return {key: _traverse(sub_data, op) for key, sub_data in data.items()} + elif isinstance(data, Tensor): + return op(data) + else: + return None + + +def _get_shapes(data): + return _traverse(data, op=lambda x: x.shape) + + +def _get_dtypes(data): + return _traverse(data, op=lambda x: x.dtype) + + +def _construct_broadcast_buffer(shapes, dtypes, device): + if isinstance(shapes, torch.Size): + return torch.empty(shapes, dtype=dtypes, device=device) + + if isinstance(shapes, (list, tuple)): + buffer = [] + for i, sub_shape in enumerate(shapes): + buffer.append(_construct_broadcast_buffer(sub_shape, dtypes[i], device)) + elif isinstance(shapes, dict): + buffer = {} + for key, sub_shape in shapes.items(): + buffer[key] = _construct_broadcast_buffer(sub_shape, dtypes[key], device) + else: + return None + return buffer + + +class SPDistForward: + """A forward tool to sync different result across sp group + + Args: + module: a function or module to process users input + sp_step: current training step to judge which rank to broadcast its result to all + name: a distinct str to save meta and async comm + comm_shape: if different ranks have different shape, mark this arg to True + device: the device for current rank, can be empty + """ + + def __init__( + self, + name: str, + comm_shape: bool, + device: torch.device = None, + ): + self.name = name + self.comm_shape = comm_shape + if device: + self.device = device + else: + self.device = get_device() + + def __call__(self, inputs) -> Any: + group = get_sequence_parallel_group() + if not group: + yield inputs + else: + device = self.device + sp_world = 1 + sp_rank = get_sequence_parallel_rank() + for local_step in range(sp_world): + src_rank = dist.get_global_rank(group, local_step) + is_src = sp_rank == local_step + local_shapes = [] + local_dtypes = [] + if local_step == 0: + local_result = inputs + _SEQ_DATA_BUF[self.name][-1] = local_result + local_shapes = _get_shapes(local_result) + local_dtypes = _get_dtypes(local_result) + if self.comm_shape: + group_shapes_lists = [None] * sp_world + dist.all_gather_object(group_shapes_lists, local_shapes, group=group) + _SEQ_DATA_META_SHAPES[self.name] = group_shapes_lists + else: + _SEQ_DATA_META_SHAPES[self.name] = [local_shapes] * sp_world + _SEQ_DATA_META_DTYPES[self.name] = local_dtypes + shapes = _SEQ_DATA_META_SHAPES[self.name][local_step] + dtypes = _SEQ_DATA_META_DTYPES[self.name] + buf_id = local_step % 2 + if local_step == 0: + sync_data = ( + local_result + if is_src + else _construct_broadcast_buffer(shapes, dtypes, device) + ) + _broadcast_data(sync_data, shapes, dtypes, src_rank, group, False) + _SEQ_DATA_BUF[self.name][buf_id] = sync_data + + # wait for async comm ops + if _SEQ_DATA_ASYNC_COMMS[self.name]: + for comm in _SEQ_DATA_ASYNC_COMMS[self.name]: + comm.wait() + # before return the sync result, do async broadcast for next batch + if local_step < sp_world - 1: + next_buf_id = 1 - buf_id + shapes = _SEQ_DATA_META_SHAPES[self.name][local_step + 1] + src_rank = dist.get_global_rank(group, local_step + 1) + is_src = sp_rank == local_step + 1 + next_sync_data = ( + _SEQ_DATA_BUF[self.name][-1] + if is_src + else _construct_broadcast_buffer(shapes, dtypes, device) + ) + _SEQ_DATA_ASYNC_COMMS[self.name] = _broadcast_data( + next_sync_data, shapes, dtypes, src_rank, group, True + ) + _SEQ_DATA_BUF[self.name][next_buf_id] = next_sync_data + yield _SEQ_DATA_BUF[self.name][buf_id] + + +sync_inputs = SPDistForward(name="bef_fwd", comm_shape=True) + + +def sync_data(data, sp_idx, name="tmp"): + group = get_sequence_parallel_group() + if group is None: + return data + # if sp_idx in _SYNC_BUFFER[name]: + # return _SYNC_BUFFER[name][sp_idx] + sp_rank = get_sequence_parallel_rank() + src_rank = dist.get_global_rank(group, sp_idx) + objects = [data] if sp_rank == sp_idx else [None] + dist.broadcast_object_list(objects, src=src_rank, group=group) + # _SYNC_BUFFER[name] = {sp_idx: objects[0]} + return objects[0] diff --git a/modules/seedvr/src/common/half_precision_fixes.py b/modules/seedvr/src/common/half_precision_fixes.py new file mode 100644 index 000000000..285f336c7 --- /dev/null +++ b/modules/seedvr/src/common/half_precision_fixes.py @@ -0,0 +1,61 @@ +import torch.nn.functional as F + + +def safe_pad_operation(x, padding, mode='constant', value=0.0): + """Safe padding operation that handles Half precision only for problematic modes""" + # Modes qui nécessitent le fix Half precision + problematic_modes = ['replicate', 'reflect', 'circular'] + + if mode in problematic_modes: + try: + return F.pad(x, padding, mode=mode, value=value) + except RuntimeError as e: + if "not implemented for 'Half'" in str(e): + original_dtype = x.dtype + return F.pad(x.float(), padding, mode=mode, value=value).to(original_dtype) + else: + raise e + else: + # Pour 'constant' et autres modes compatibles, pas de fix nécessaire + return F.pad(x, padding, mode=mode, value=value) + + +def safe_interpolate_operation(x, size=None, scale_factor=None, mode='nearest', align_corners=None, recompute_scale_factor=None): + """Safe interpolate operation that handles Half precision for problematic modes""" + # Modes qui peuvent causer des problèmes avec Half precision + problematic_modes = ['bilinear', 'bicubic', 'trilinear'] + + if mode in problematic_modes: + try: + return F.interpolate( + x, + size=size, + scale_factor=scale_factor, + mode=mode, + align_corners=align_corners, + recompute_scale_factor=recompute_scale_factor + ) + except RuntimeError as e: + if ("not implemented for 'Half'" in str(e) or + "compute_indices_weights" in str(e)): + original_dtype = x.dtype + return F.interpolate( + x.float(), + size=size, + scale_factor=scale_factor, + mode=mode, + align_corners=align_corners, + recompute_scale_factor=recompute_scale_factor + ).to(original_dtype) + else: + raise e + else: + # Pour 'nearest' et autres modes compatibles, pas de fix nécessaire + return F.interpolate( + x, + size=size, + scale_factor=scale_factor, + mode=mode, + align_corners=align_corners, + recompute_scale_factor=recompute_scale_factor + ) diff --git a/modules/seedvr/src/common/logger.py b/modules/seedvr/src/common/logger.py new file mode 100644 index 000000000..48b6eab67 --- /dev/null +++ b/modules/seedvr/src/common/logger.py @@ -0,0 +1,44 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Logging utility functions. +""" + +import logging +import sys +from typing import Optional +from .distributed import get_global_rank, get_local_rank, get_world_size + + +_default_handler = logging.StreamHandler(sys.stdout) +_default_handler.setFormatter( + logging.Formatter( + "%(asctime)s " + + (f"[Rank:{get_global_rank()}]" if get_world_size() > 1 else "") + + (f"[LocalRank:{get_local_rank()}]" if get_world_size() > 1 else "") + + "[%(threadName).12s][%(name)s][%(levelname).5s] " + + "%(message)s" + ) +) + + +def get_logger(name: Optional[str] = None) -> logging.Logger: + """ + Get a logger. + """ + logger = logging.getLogger(name) + logger.addHandler(_default_handler) + logger.setLevel(logging.INFO) + return logger diff --git a/modules/seedvr/src/common/partition.py b/modules/seedvr/src/common/partition.py new file mode 100644 index 000000000..648c87fe2 --- /dev/null +++ b/modules/seedvr/src/common/partition.py @@ -0,0 +1,59 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +""" +Partition utility functions. +""" + +from typing import Any, List + + +def partition_by_size(data: List[Any], size: int) -> List[List[Any]]: + """ + Partition a list by size. + When indivisible, the last group contains fewer items than the target size. + + Examples: + - data: [1,2,3,4,5] + - size: 2 + - return: [[1,2], [3,4], [5]] + """ + assert size > 0 + return [data[i : (i + size)] for i in range(0, len(data), size)] + + +def partition_by_groups(data: List[Any], groups: int) -> List[List[Any]]: + """ + Partition a list by groups. + When indivisible, some groups may have more items than others. + + Examples: + - data: [1,2,3,4,5] + - groups: 2 + - return: [[1,3,5], [2,4]] + """ + assert groups > 0 + return [data[i::groups] for i in range(groups)] + + +def shift_list(data: List[Any], n: int) -> List[Any]: + """ + Rotate a list by n elements. + + Examples: + - data: [1,2,3,4,5] + - n: 3 + - return: [4,5,1,2,3] + """ + return data[(n % len(data)) :] + data[: (n % len(data))] diff --git a/modules/seedvr/src/common/seed.py b/modules/seedvr/src/common/seed.py new file mode 100644 index 000000000..2469ad944 --- /dev/null +++ b/modules/seedvr/src/common/seed.py @@ -0,0 +1,28 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +import random +from typing import Optional +import numpy as np +import torch +from .distributed import get_global_rank + + +def set_seed(seed: Optional[int], same_across_ranks: bool = False): + """Function that sets the seed for pseudo-random number generators.""" + if seed is not None: + seed += get_global_rank() if not same_across_ranks else 0 + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) diff --git a/modules/seedvr/src/core/__init__.py b/modules/seedvr/src/core/__init__.py new file mode 100644 index 000000000..e284d32ed --- /dev/null +++ b/modules/seedvr/src/core/__init__.py @@ -0,0 +1,47 @@ +""" +Core Module for SeedVR2 + +Contains the main business logic and model management functionality: +- Model configuration and loading +- Architecture detection and memory estimation +- Runner creation and management +- Generation pipeline and logic +""" +''' +from .model_manager import ( + configure_runner, + load_quantized_state_dict, + configure_dit_model_inference, + configure_vae_model_inference, +) + +from .generation import ( + generation_step, + generation_loop, + cut_videos, + prepare_video_transforms, + load_text_embeddings, + calculate_optimal_batch_params +) + +from .infer import VideoDiffusionInfer + +__all__ = [ + # Model management + 'configure_runner', + 'load_quantized_state_dict', + 'configure_dit_model_inference', + 'configure_vae_model_inference', + + # Generation logic + 'generation_step', + 'generation_loop', + 'cut_videos', + 'prepare_video_transforms', + 'load_text_embeddings', + 'calculate_optimal_batch_params', + + # Infer + 'VideoDiffusionInfer' +] +''' diff --git a/modules/seedvr/src/core/generation.py b/modules/seedvr/src/core/generation.py new file mode 100644 index 000000000..401d8b874 --- /dev/null +++ b/modules/seedvr/src/core/generation.py @@ -0,0 +1,338 @@ +import torch +from torchvision.transforms import Compose, Lambda, Normalize +from ..optimization.performance import optimized_video_rearrange, optimized_single_video_rearrange, optimized_sample_to_image_format +from ..common.seed import set_seed +from ..data.image.transforms.divisible_crop import DivisibleCrop +from ..data.image.transforms.na_resize import NaResize +from ..utils.color_fix import wavelet_reconstruction + + + +def generation_step(runner, text_embeds_dict, cond_latents, temporal_overlap, device): + """ + Execute a single generation step with adaptive dtype handling + + Args: + runner: VideoDiffusionInfer instance + text_embeds_dict (dict): Text embeddings for positive and negative prompts + cond_latents (list): Conditional latents for generation + temporal_overlap (int): Number of frames for temporal overlap + + Returns: + tuple: (samples, last_latents) for potential temporal continuation + + Features: + - Adaptive dtype detection (FP8/FP16/BFloat16) + - Optimal autocast configuration for each model type + - Memory-efficient noise generation and reuse + - Automatic device placement with dtype preservation + - Advanced inference optimization + """ + # Adaptive dtype detection for optimal performance + model_dtype = next(runner.dit.parameters()).dtype + + # Configure dtypes according to model architecture + if model_dtype in (torch.float8_e4m3fn, torch.float8_e5m2): + # FP8 native: use BFloat16 for intermediate calculations (optimal compatibility) + dtype = torch.bfloat16 + elif model_dtype == torch.float16: + dtype = torch.float16 + else: + dtype = torch.bfloat16 + + def _move_to_cuda(x): + """Move tensors to CUDA with adaptive optimal dtype""" + return [i.to(device, dtype=dtype) for i in x] + + # Memory optimization: Generate noise once and reuse to save VRAM + with torch.cuda.device(device): + base_noise = torch.randn_like(cond_latents[0], dtype=dtype) + noises = [base_noise] + aug_noises = [base_noise * 0.1 + torch.randn_like(base_noise) * 0.05] + + # Move tensors with adaptive dtype (optimized for FP8/FP16/BFloat16) + noises, aug_noises, cond_latents = _move_to_cuda(noises), _move_to_cuda(aug_noises), _move_to_cuda(cond_latents) + + cond_noise_scale = 0.0 + + def _add_noise(x, aug_noise): + # Use adaptive optimal dtype + t = ( + torch.tensor([1000.0], device=device, dtype=dtype) + * cond_noise_scale + ) + shape = torch.tensor(x.shape[1:], device=device)[None] + t = runner.timestep_transform(t, shape) + x = runner.schedule.forward(x, aug_noise, t) + return x + + condition = runner.get_condition( + noises[0], + task="sr", + latent_blur=_add_noise(cond_latents[0], aug_noises[0]), + ) + conditions = [condition] + + with torch.no_grad(): + # Use adaptive autocast for optimal performance + video_tensors = runner.inference( + noises=noises, + conditions=conditions, + temporal_overlap=temporal_overlap, + **text_embeds_dict, + ) + + # Process samples with advanced optimization + samples = optimized_video_rearrange(video_tensors) + noises = noises[0].to("cpu") + aug_noises = aug_noises[0].to("cpu") + cond_latents = cond_latents[0].to("cpu") + conditions = conditions[0].to("cpu") + condition = condition.to("cpu") + del noises, aug_noises, cond_latents, conditions, condition + + return samples #, last_latents + + +def cut_videos(videos): + t = videos.size(1) + + if t % 4 == 1: + return videos + + padding_needed = (4 - (t % 4)) % 4 + 1 + last_frame = videos[:, -1:].expand(-1, padding_needed, -1, -1).contiguous() + result = torch.cat([videos, last_frame], dim=1) + return result + + +def generation_loop(runner, images, cfg_scale=1.0, seed=666, res_w=720, batch_size=90, temporal_overlap=0, progress_callback=None, device:str='cpu'): + """ + Main generation loop with context-aware temporal processing + + Args: + runner: VideoDiffusionInfer instance + images (torch.Tensor): Input images for upscaling + cfg_scale (float): Classifier-free guidance scale + seed (int): Random seed for reproducibility + res_w (int): Target resolution width + batch_size (int): Batch size for processing + temporal_overlap (int): Frames for temporal continuity + progress_callback (callable): Optional callback for progress reporting + + Returns: + torch.Tensor: Generated video frames + + Features: + - Context-aware generation with temporal overlap + - Adaptive dtype pipeline (FP8/FP16/BFloat16) + - Memory-optimized batch processing + - Advanced video transformation pipeline + - Intelligent VRAM management throughout process + - Real-time progress reporting + """ + model_dtype = None + model_dtype = next(runner.dit.parameters()).dtype + compute_dtype = model_dtype + + # Configure classifier-free guidance + runner.config.diffusion.cfg.scale = cfg_scale + runner.config.diffusion.cfg.rescale = 0.0 + # Configure sampling steps + runner.config.diffusion.timesteps.sampling.steps = 1 + runner.configure_diffusion() + + # Set random seed + set_seed(seed) + + # Advanced video transformation pipeline + video_transform = Compose([ + NaResize( + resolution=(res_w), + mode="side", + downsample_only=False, + ), + Lambda(lambda x: torch.clamp(x, 0.0, 1.0)), + DivisibleCrop((16, 16)), + Normalize(0.5, 0.5), + Lambda(lambda x: x.permute(1, 0, 2, 3)), # t c h w -> c t h w (faster than Rearrange) + ]) + + # Initialize generation state + batch_samples = [] + + # Load text embeddings with adaptive dtype + text_embeds = {"texts_pos": [runner.text_pos_embeds], "texts_neg": [runner.text_neg_embeds]} + + # Calculate processing parameters + step = batch_size - temporal_overlap + if step <= 0: + step = batch_size + temporal_overlap = 0 + + # Calculate total batches for progress reporting + total_batches = len(range(0, len(images), step)) + + # Main processing loop with context awareness + for batch_count, batch_idx in enumerate(range(0, len(images), step)): + # Calculate batch indices with overlap + if batch_idx == 0: + # First batch: no overlap + start_idx = 0 + end_idx = min(batch_size, len(images)) + effective_batch_size = end_idx - start_idx + else: + # Subsequent batches: temporal overlap + start_idx = batch_idx + end_idx = min(start_idx + batch_size, len(images)) + effective_batch_size = end_idx - start_idx + if effective_batch_size <= temporal_overlap: + break # Not enough new frames, stop + + current_frames = end_idx - start_idx + + # Process current batch + video = images[start_idx:end_idx] + # Use adaptive computation dtype + video = video.permute(0, 3, 1, 2).to(device, dtype=compute_dtype) + + # Apply video transformations with memory optimization + transformed_video = video_transform(video) + del video + #video = video.to("cpu") + #del video + ori_lengths = [transformed_video.size(1)] + + # Handle correct format: frames % 4 == 1 + t = transformed_video.size(1) + + if len(images) >= 5 and t % 4 != 1: + transformed_video = cut_videos(transformed_video) + + # Context-aware temporal strategy + # First batch: standard complete diffusion + cond_latents = runner.vae_encode([transformed_video]) + + # Normal generation + samples = generation_step(runner, text_embeds, cond_latents=cond_latents, temporal_overlap=temporal_overlap, device=device) + #del cond_latents + del cond_latents + + # Post-process samples + sample = samples[0] + del samples + #del samples + if ori_lengths[0] < sample.shape[0]: + sample = sample[:ori_lengths[0]] + + # Apply color correction if available + transformed_video = transformed_video.to(device) + input_video = [optimized_single_video_rearrange(transformed_video)] + del transformed_video + sample = wavelet_reconstruction(sample, input_video[0][:sample.size(0)]) + del input_video + + # Convert to final image format + sample = optimized_sample_to_image_format(sample) + sample = sample.clip(-1, 1).mul_(0.5).add_(0.5) + sample_cpu = sample.to(torch.float16).to("cpu") + del sample + batch_samples.append(sample_cpu) + #del sample + + if progress_callback: + progress_callback(batch_count+1, total_batches, current_frames, "Processing batch...") + + + # 1. Calculer la taille totale finale + total_frames = sum(batch.shape[0] for batch in batch_samples) + if len(batch_samples) > 0: + sample_shape = batch_samples[0].shape + H, W, C = sample_shape[1], sample_shape[2], sample_shape[3] + final_video_images = torch.empty((total_frames, H, W, C), dtype=torch.float16) + block_size = 500 + current_idx = 0 + + for block_start in range(0, len(batch_samples), block_size): + block_end = min(block_start + block_size, len(batch_samples)) + current_block = [] + for i in range(block_start, block_end): + current_block.append(batch_samples[i].to(device)) + block_result = torch.cat(current_block, dim=0) + block_frames = block_result.shape[0] + final_video_images[current_idx:current_idx + block_frames] = block_result.to("cpu") + current_idx += block_frames + del current_block, block_result + else: + print("SeedVR2: No batch_samples to process") + final_video_images = torch.empty((0, 0, 0, 0), dtype=torch.float16) + + return final_video_images + + +def prepare_video_transforms(res_w): + """ + Prepare optimized video transformation pipeline + + Args: + res_w (int): Target resolution width + + Returns: + Compose: Configured transformation pipeline + + Features: + - Resolution-aware upscaling (no downsampling) + - Proper normalization for model compatibility + - Memory-efficient tensor operations + """ + return Compose([ + NaResize( + resolution=(res_w), + mode="side", + downsample_only=False, # Model trained for high resolution + ), + Lambda(lambda x: torch.clamp(x, 0.0, 1.0)), + DivisibleCrop((16, 16)), + Normalize(0.5, 0.5), + Lambda(lambda x: x.permute(1, 0, 2, 3)), # t c h w -> c t h w + ]) + + +def calculate_optimal_batch_params(total_frames, batch_size, temporal_overlap): + """ + Calculate optimal batch processing parameters + + Args: + total_frames (int): Total number of frames + batch_size (int): Desired batch size + temporal_overlap (int): Temporal overlap frames + + Returns: + dict: Optimized parameters and recommendations + + Features: + - 4n+1 constraint optimization + - Padding waste calculation + - Performance recommendations + """ + step = batch_size - temporal_overlap + if step <= 0: + step = batch_size + temporal_overlap = 0 + + # Find optimal batch sizes (4n+1 constraint) + optimal_batches = [x for x in [i for i in range(1, 200) if i % 4 == 1] if x <= total_frames] + best_batch = max(optimal_batches) if optimal_batches else 1 + + # Calculate potential padding waste + padding_waste = 0 + if batch_size not in optimal_batches: + padding_waste = sum(((i // 4) + 1) * 4 + 1 - i for i in range(batch_size, total_frames, batch_size)) + + return { + 'step': step, + 'temporal_overlap': temporal_overlap, + 'best_batch': best_batch, + 'padding_waste': padding_waste, + 'is_optimal': batch_size in optimal_batches + } diff --git a/modules/seedvr/src/core/infer.py b/modules/seedvr/src/core/infer.py new file mode 100644 index 000000000..7882d3a61 --- /dev/null +++ b/modules/seedvr/src/core/infer.py @@ -0,0 +1,323 @@ +from typing import List, Optional, Tuple, Union +import torch +from einops import rearrange +from omegaconf import DictConfig, ListConfig +from ..common.diffusion import classifier_free_guidance_dispatcher, create_sampler_from_config, create_sampling_timesteps_from_config, create_schedule_from_config +from ..models.dit_v2 import na + + +def optimized_channels_to_last(tensor: torch.Tensor) -> torch.Tensor: + """🚀 Optimized replacement for rearrange(tensor, 'b c ... -> b ... c') + Moves channels from position 1 to last position using PyTorch native operations. + """ + if tensor.ndim == 3: # [batch, channels, spatial] + return tensor.permute(0, 2, 1) + elif tensor.ndim == 4: # [batch, channels, height, width] + return tensor.permute(0, 2, 3, 1) + elif tensor.ndim == 5: # [batch, channels, depth, height, width] + return tensor.permute(0, 2, 3, 4, 1) + else: + # Fallback for other dimensions - move channel (dim=1) to last + dims = list(range(tensor.ndim)) + dims = [dims[0]] + dims[2:] + [dims[1]] # [0, 2, 3, ..., 1] + return tensor.permute(*dims) + +def optimized_channels_to_second(tensor): + """🚀 Optimized replacement for rearrange(tensor, 'b ... c -> b c ...') + Moves channels from last position to position 1 using PyTorch native operations. + """ + if tensor.ndim == 3: # [batch, spatial, channels] + return tensor.permute(0, 2, 1) + elif tensor.ndim == 4: # [batch, height, width, channels] + return tensor.permute(0, 3, 1, 2) + elif tensor.ndim == 5: # [batch, depth, height, width, channels] + return tensor.permute(0, 4, 1, 2, 3) + else: + # Fallback for other dimensions - move last dim to position 1 + dims = list(range(tensor.ndim)) + dims = [dims[0], dims[-1]] + dims[1:-1] # [0, -1, 1, 2, ..., -2] + return tensor.permute(*dims) + + +class VideoDiffusionInfer(): + def __init__(self, config: DictConfig, device: str, dtype: torch.dtype): + self.config = config + self.device = device + self.dtype = dtype + self.vae = None + self.dit = None + self.sampler = None + self.schedule = None + def get_condition(self, latent: torch.Tensor, latent_blur: torch.Tensor, task: str) -> torch.Tensor: + t, h, w, c = latent.shape + cond = torch.zeros([t, h, w, c + 1], device=latent.device, dtype=latent.dtype) + if task == "t2v" or t == 1: + # t2i or t2v generation. + if task == "sr": + cond[:, ..., :-1] = latent_blur[:] + cond[:, ..., -1:] = 1.0 + return cond + if task == "i2v": + # i2v generation. + cond[:1, ..., :-1] = latent[:1] + cond[:1, ..., -1:] = 1.0 + return cond + if task == "v2v": + # v2v frame extension. + cond[:2, ..., :-1] = latent[:2] + cond[:2, ..., -1:] = 1.0 + return cond + if task == "sr": + # sr generation. + cond[:, ..., :-1] = latent_blur[:] + cond[:, ..., -1:] = 1.0 + return cond + raise NotImplementedError + + def configure_diffusion(self): + self.schedule = create_schedule_from_config( + config=self.config.diffusion.schedule, + ) + self.sampling_timesteps = create_sampling_timesteps_from_config( # pylint: disable=attribute-defined-outside-init + config=self.config.diffusion.timesteps.sampling, + schedule=self.schedule, + device=self.device, + ) + self.sampler = create_sampler_from_config( + config=self.config.diffusion.sampler, + schedule=self.schedule, + timesteps=self.sampling_timesteps, + ) + + # -------------------------------- Helper ------------------------------- # + + @torch.no_grad() + def vae_encode(self, samples: List[torch.Tensor]) -> List[torch.Tensor]: + use_sample = self.config.vae.get("use_sample", True) + latents = [] + if len(samples) > 0: + dtype = self.vae.dtype + scale = self.config.vae.scaling_factor + shift = self.config.vae.get("shifting_factor", 0.0) + + if isinstance(scale, ListConfig): + scale = torch.tensor(scale, device=self.device, dtype=dtype) + if isinstance(shift, ListConfig): + shift = torch.tensor(shift, device=self.device, dtype=dtype) + + # Group samples of the same shape to batches if enabled. + if self.config.vae.grouping: + batches, indices = na.pack(samples) + else: + batches = [sample.unsqueeze(0) for sample in samples] + + # Vae process by each group. + for sample in batches: + sample = sample.to(self.device, dtype) + if hasattr(self.vae, "preprocess"): + sample = self.vae.preprocess(sample) + if use_sample: + latent = self.vae.encode(sample).latent + else: + # Deterministic vae encode, only used for i2v inference (optionally) + latent = self.vae.encode(sample).posterior.mode().squeeze(2) + latent = latent.unsqueeze(2) if latent.ndim == 4 else latent + latent = rearrange(latent, "b c ... -> b ... c") + #latent = optimized_channels_to_last(latent) + latent = (latent - shift) * scale + latents.append(latent) + + # Ungroup back to individual latent with the original order. + if self.config.vae.grouping: + latents = na.unpack(latents, indices) + else: + latents = [latent.squeeze(0) for latent in latents] + return latents + + + @torch.no_grad() + def vae_decode(self, latents: List[torch.Tensor], target_dtype: torch.dtype = None) -> List[torch.Tensor]: + """🚀 VAE decode optimisé - décodage direct sans chunking, compatible avec autocast externe""" + samples = [] + if len(latents) > 0: + device = self.device + dtype = self.vae.dtype + scale = self.config.vae.scaling_factor + shift = self.config.vae.get("shifting_factor", 0.0) + + if isinstance(scale, ListConfig): + scale = torch.tensor(scale, device=device, dtype=dtype) + if isinstance(shift, ListConfig): + shift = torch.tensor(shift, device=device, dtype=dtype) + + + # 🚀 OPTIMISATION 1: Group latents intelligemment pour batch processing + if self.config.vae.grouping: + latents, indices = na.pack(latents) + else: + latents = [latent.unsqueeze(0) for latent in latents] + + # 🚀 OPTIMISATION 2: Traitement batch optimisé avec dtype adaptatif + for _i, latent in enumerate(latents): + # Préparation optimisée du latent + # Utiliser target_dtype si fourni (évite double autocast) + effective_dtype = target_dtype if target_dtype is not None else dtype + latent = latent.to(device, effective_dtype, non_blocking=True) + latent = latent / scale + shift + latent = rearrange(latent, "b ... c -> b c ...") + #latent = optimized_channels_to_second(latent) + latent = latent.squeeze(2) + + # 🚀 OPTIMISATION 3: Décodage direct SANS autocast (utilise l'autocast externe) + sample = self.vae.decode(latent).sample + #sample = self.vae.decode(latent).sample + #sample = self.vae.decode(latent).sample + + # 🚀 OPTIMISATION 4: Post-processing conditionnel + if hasattr(self.vae, "postprocess"): + sample = self.vae.postprocess(sample) + + samples.append(sample) + + # Ungroup back to individual sample with the original order. + if self.config.vae.grouping: + samples = na.unpack(samples, indices) + else: + samples = [sample.squeeze(0) for sample in samples] + return samples + + def timestep_transform(self, timesteps: torch.Tensor, latents_shapes: torch.Tensor): + # Skip if not needed. + if not self.config.diffusion.timesteps.get("transform", False): + return timesteps + + # Compute resolution. + vt = self.config.vae.model.get("temporal_downsample_factor", 4) + vs = self.config.vae.model.get("spatial_downsample_factor", 8) + frames = (latents_shapes[:, 0] - 1) * vt + 1 + heights = latents_shapes[:, 1] * vs + widths = latents_shapes[:, 2] * vs + + # Compute shift factor. + def get_lin_function(x1, y1, x2, y2): + m = (y2 - y1) / (x2 - x1) + b = y1 - m * x1 + return lambda x: m * x + b + + img_shift_fn = get_lin_function(x1=256 * 256, y1=1.0, x2=1024 * 1024, y2=3.2) + vid_shift_fn = get_lin_function(x1=256 * 256 * 37, y1=1.0, x2=1280 * 720 * 145, y2=5.0) + shift = torch.where( + frames > 1, + vid_shift_fn(heights * widths * frames), + img_shift_fn(heights * widths), + ) + + # Shift timesteps. + timesteps = timesteps / self.schedule.T + timesteps = shift * timesteps / (1 + (shift - 1) * timesteps) + timesteps = timesteps * self.schedule.T + return timesteps + + @torch.no_grad() + def inference( + self, + noises: List[torch.Tensor], + conditions: List[torch.Tensor], + texts_pos: Union[List[str], List[torch.Tensor], List[Tuple[torch.Tensor]]], + texts_neg: Union[List[str], List[torch.Tensor], List[Tuple[torch.Tensor]]], + cfg_scale: Optional[float] = None, + temporal_overlap: int = 0, # pylint: disable=unused-argument + ) -> List[torch.Tensor]: + assert len(noises) == len(conditions) == len(texts_pos) == len(texts_neg) + batch_size = len(noises) + + # Return if empty. + if batch_size == 0: + return [] + + # Set cfg scale + if cfg_scale is None: + cfg_scale = self.config.diffusion.cfg.scale + + # 🚀 OPTIMISATION: Détecter le dtype du modèle pour performance optimale + model_dtype = next(self.dit.parameters()).dtype + # Adapter les dtypes selon le modèle + if model_dtype in (torch.float8_e4m3fn, torch.float8_e5m2): + target_dtype = torch.float16 + elif model_dtype == torch.float16: + target_dtype = torch.float16 + else: + target_dtype = torch.bfloat16 + # Text embeddings. + assert type(texts_pos[0]) is type(texts_neg[0]) + if isinstance(texts_pos[0], str): + text_pos_embeds, text_pos_shapes = self.text_encode(texts_pos) # pylint: disable=no-member + text_neg_embeds, text_neg_shapes = self.text_encode(texts_neg) # pylint: disable=no-member + elif isinstance(texts_pos[0], tuple): + text_pos_embeds, text_pos_shapes = [], [] + text_neg_embeds, text_neg_shapes = [], [] + for pos in zip(*texts_pos): + emb, shape = na.flatten(pos) + text_pos_embeds.append(emb) + text_pos_shapes.append(shape) + for neg in zip(*texts_neg): + emb, shape = na.flatten(neg) + text_neg_embeds.append(emb) + text_neg_shapes.append(shape) + else: + text_pos_embeds, text_pos_shapes = na.flatten(texts_pos) + text_neg_embeds, text_neg_shapes = na.flatten(texts_neg) + + # Adapter les embeddings texte au dtype cible (compatible avec FP8) + if isinstance(text_pos_embeds, torch.Tensor): + text_pos_embeds = text_pos_embeds.to(target_dtype) + if isinstance(text_neg_embeds, torch.Tensor): + text_neg_embeds = text_neg_embeds.to(target_dtype) + + # Flatten. + latents, latents_shapes = na.flatten(noises) + latents_cond, _ = na.flatten(conditions) + + # Adapter les latents au dtype cible (compatible avec FP8) + latents = latents.to(target_dtype) if latents.dtype != target_dtype else latents + latents_cond = latents_cond.to(target_dtype) if latents_cond.dtype != target_dtype else latents_cond + self.dit = self.dit.to(device=self.device, dtype=target_dtype) + + latents = self.sampler.sample( + x=latents, + f=lambda args: classifier_free_guidance_dispatcher( + pos=lambda: self.dit( + vid=torch.cat([args.x_t, latents_cond], dim=-1), + txt=text_pos_embeds, + vid_shape=latents_shapes, + txt_shape=text_pos_shapes, + timestep=args.t.repeat(batch_size), + ).vid_sample, + neg=lambda: self.dit( + vid=torch.cat([args.x_t, latents_cond], dim=-1), + txt=text_neg_embeds, + vid_shape=latents_shapes, + txt_shape=text_neg_shapes, + timestep=args.t.repeat(batch_size), + ).vid_sample, + scale=( + cfg_scale + if (args.i + 1) / len(self.sampler.timesteps) + <= self.config.diffusion.cfg.get("partial", 1) + else 1.0 + ), + rescale=self.config.diffusion.cfg.rescale, + ), + ) + + latents = na.unflatten(latents, latents_shapes) + + # 🎯 Pré-calcul des dtypes (une seule fois) + vae_dtype = self.vae.dtype + decode_dtype = torch.float16 if (vae_dtype == torch.float16 or target_dtype == torch.float16) else vae_dtype + samples = self.vae_decode(latents, target_dtype=decode_dtype) + + if samples and len(samples) > 0 and samples[0].dtype != torch.float16: + samples = [sample.to(torch.float16, non_blocking=True) for sample in samples] + + return samples diff --git a/modules/seedvr/src/core/model_manager.py b/modules/seedvr/src/core/model_manager.py new file mode 100644 index 000000000..3abade938 --- /dev/null +++ b/modules/seedvr/src/core/model_manager.py @@ -0,0 +1,57 @@ +import os +import torch +from omegaconf import OmegaConf +from safetensors.torch import load_file as load_safetensors_file +from huggingface_hub import hf_hub_download +from ..optimization.memory_manager import preinitialize_rope_cache +from ..common.config import load_config, create_object +from ..core.infer import VideoDiffusionInfer + + +def configure_runner(model_name, cache_dir, device:str='cpu', dtype:torch.dtype=None): + repo_id = "vladmandic/SeedVR2" + script_directory = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + config_path = os.path.join(script_directory, './config_7b.yaml') if "7b" in model_name else os.path.join(script_directory, './config_3b.yaml') + config = load_config(config_path) + + runner = VideoDiffusionInfer(config, device=device, dtype=dtype) + OmegaConf.set_readonly(runner.config, False) + + # load dit + with torch.device("meta"): + runner.dit = create_object(config.dit.model) + runner.dit.requires_grad_(False).eval() + runner.dit.to_empty(device="cpu") + model_file = hf_hub_download(repo_id=repo_id, filename=model_name, cache_dir=cache_dir) + state_dict = load_safetensors_file(model_file) + runner.dit.load_state_dict(state_dict, assign=True) + runner.dit = runner.dit.to(device="cpu", dtype=dtype) + del state_dict + + # load vae + vae_config_path = os.path.join(script_directory, 'src/models/video_vae_v3/s8_c16_t4_inflation_sd3.yaml') + vae_config = OmegaConf.load(vae_config_path) + config.vae.model = OmegaConf.merge(config.vae.model, vae_config) + + vae_file = hf_hub_download(repo_id=repo_id, filename=config.vae.checkpoint, cache_dir=cache_dir) + with torch.device("meta"): + runner.vae = create_object(config.vae.model) + runner.vae.requires_grad_(False).eval() + runner.vae.to_empty(device="cpu") + state_dict = load_safetensors_file(vae_file) + runner.vae.load_state_dict(state_dict) + runner.vae = runner.vae.to(device="cpu", dtype=dtype) + runner.config.vae.dtype = str(dtype) + runner.config.vae.slicing = {'split_size': 8, 'memory_device': 'same'} + runner.config.vae.memory_limit = {'conv_max_mem': 0.2, 'norm_max_mem': 0.2} + runner.vae.set_causal_slicing(**runner.config.vae.slicing) + runner.vae.set_memory_limit(**runner.config.vae.memory_limit) + del state_dict + + # load embeds + pos_embeds_file = hf_hub_download(repo_id=repo_id, filename='pos_emb.pt', cache_dir=cache_dir) + neg_embeds_file = hf_hub_download(repo_id=repo_id, filename='neg_emb.pt', cache_dir=cache_dir) + runner.text_pos_embeds = torch.load(pos_embeds_file).to(device=device, dtype=dtype) + runner.text_neg_embeds = torch.load(neg_embeds_file).to(device=device, dtype=dtype) + + return runner diff --git a/modules/seedvr/src/data/image/transforms/area_resize.py b/modules/seedvr/src/data/image/transforms/area_resize.py new file mode 100644 index 000000000..dd3ca9d4b --- /dev/null +++ b/modules/seedvr/src/data/image/transforms/area_resize.py @@ -0,0 +1,131 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +import math +import random +from typing import Union +import torch +from PIL import Image +from torchvision.transforms import functional as TVF +from torchvision.transforms.functional import InterpolationMode + + +class AreaResize: + def __init__( + self, + max_area: float, + downsample_only: bool = False, + interpolation: InterpolationMode = InterpolationMode.BICUBIC, + ): + self.max_area = max_area + self.downsample_only = downsample_only + self.interpolation = interpolation + + def __call__(self, image: Union[torch.Tensor, Image.Image]): + + if isinstance(image, torch.Tensor): + height, width = image.shape[-2:] + elif isinstance(image, Image.Image): + width, height = image.size + else: + raise NotImplementedError + + scale = math.sqrt(self.max_area / (height * width)) + + # keep original height and width for small pictures. + scale = 1 if scale >= 1 and self.downsample_only else scale + + resized_height, resized_width = round(height * scale), round(width * scale) + + return TVF.resize( + image, + size=(resized_height, resized_width), + interpolation=self.interpolation, + ) + + +class AreaRandomCrop: + def __init__( + self, + max_area: float, + ): + self.max_area = max_area + + def get_params(self, input_size, output_size): + """Get parameters for ``crop`` for a random crop. + + Args: + img (PIL Image): Image to be cropped. + output_size (tuple): Expected output size of the crop. + + Returns: + tuple: params (i, j, h, w) to be passed to ``crop`` for random crop. + """ + # w, h = _get_image_size(img) + h, w = input_size + th, tw = output_size + if w <= tw and h <= th: + return 0, 0, h, w + + i = random.randint(0, h - th) + j = random.randint(0, w - tw) + return i, j, th, tw + + def __call__(self, image: Union[torch.Tensor, Image.Image]): + if isinstance(image, torch.Tensor): + height, width = image.shape[-2:] + elif isinstance(image, Image.Image): + width, height = image.size + else: + raise NotImplementedError + + resized_height = math.sqrt(self.max_area / (width / height)) + resized_width = (width / height) * resized_height + + resized_height, resized_width = round(resized_height), round(resized_width) + i, j, h, w = self.get_params((height, width), (resized_height, resized_width)) + image = TVF.crop(image, i, j, h, w) + return image + +class ScaleResize: + def __init__( + self, + scale: float, + ): + self.scale = scale + + def __call__(self, image: Union[torch.Tensor, Image.Image]): + if isinstance(image, torch.Tensor): + height, width = image.shape[-2:] + interpolation_mode = InterpolationMode.BILINEAR + antialias = True if image.ndim == 4 else "warn" + elif isinstance(image, Image.Image): + width, height = image.size + interpolation_mode = InterpolationMode.LANCZOS + antialias = "warn" + else: + raise NotImplementedError + + scale = self.scale + + # keep original height and width for small pictures + + resized_height, resized_width = round(height * scale), round(width * scale) + image = TVF.resize( + image, + size=(resized_height, resized_width), + interpolation=interpolation_mode, + antialias=antialias, + ) + return image diff --git a/modules/seedvr/src/data/image/transforms/divisible_crop.py b/modules/seedvr/src/data/image/transforms/divisible_crop.py new file mode 100644 index 000000000..d1815b03e --- /dev/null +++ b/modules/seedvr/src/data/image/transforms/divisible_crop.py @@ -0,0 +1,40 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Union +import torch +from PIL import Image +from torchvision.transforms import functional as TVF + + +class DivisibleCrop: + def __init__(self, factor): + if not isinstance(factor, tuple): + factor = (factor, factor) + + self.height_factor, self.width_factor = factor[0], factor[1] + + def __call__(self, image: Union[torch.Tensor, Image.Image]): + if isinstance(image, torch.Tensor): + height, width = image.shape[-2:] + elif isinstance(image, Image.Image): + width, height = image.size + else: + raise NotImplementedError + + cropped_height = height - (height % self.height_factor) + cropped_width = width - (width % self.width_factor) + + image = TVF.center_crop(img=image, output_size=(cropped_height, cropped_width)) + return image diff --git a/modules/seedvr/src/data/image/transforms/na_resize.py b/modules/seedvr/src/data/image/transforms/na_resize.py new file mode 100644 index 000000000..d230e25e3 --- /dev/null +++ b/modules/seedvr/src/data/image/transforms/na_resize.py @@ -0,0 +1,50 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Literal +from torchvision.transforms import CenterCrop, Compose, InterpolationMode, Resize + +from .area_resize import AreaResize +from .side_resize import SideResize + + +def NaResize( + resolution: int, + mode: Literal["area", "side"], + downsample_only: bool, + interpolation: InterpolationMode = InterpolationMode.BICUBIC, +): + if mode == "area": + return AreaResize( + max_area=resolution**2, + downsample_only=downsample_only, + interpolation=interpolation, + ) + if mode == "side": + return SideResize( + size=resolution, + downsample_only=downsample_only, + interpolation=interpolation, + ) + if mode == "square": + return Compose( + [ + Resize( + size=resolution, + interpolation=interpolation, + ), + CenterCrop(resolution), + ] + ) + raise ValueError(f"Unknown resize mode: {mode}") diff --git a/modules/seedvr/src/data/image/transforms/side_resize.py b/modules/seedvr/src/data/image/transforms/side_resize.py new file mode 100644 index 000000000..6e07402b2 --- /dev/null +++ b/modules/seedvr/src/data/image/transforms/side_resize.py @@ -0,0 +1,54 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Union +import torch +from PIL import Image +from torchvision.transforms import InterpolationMode +from torchvision.transforms import functional as TVF + + +class SideResize: + def __init__( + self, + size: int, + downsample_only: bool = False, + interpolation: InterpolationMode = InterpolationMode.BICUBIC, + ): + self.size = size + self.downsample_only = downsample_only + self.interpolation = interpolation + + def __call__(self, image: Union[torch.Tensor, Image.Image]): + """ + Args: + image (PIL Image or Tensor): Image to be scaled. + + Returns: + PIL Image or Tensor: Rescaled image. + """ + if isinstance(image, torch.Tensor): + height, width = image.shape[-2:] + elif isinstance(image, Image.Image): + width, height = image.size + else: + raise NotImplementedError + + if self.downsample_only and min(width, height) < self.size: + # keep original height and width for small pictures. + size = min(width, height) + else: + size = self.size + + return TVF.resize(image, size, self.interpolation) diff --git a/modules/seedvr/src/models/dit/attention.py b/modules/seedvr/src/models/dit/attention.py new file mode 100644 index 000000000..b0dda518a --- /dev/null +++ b/modules/seedvr/src/models/dit/attention.py @@ -0,0 +1,94 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +import torch +import torch.nn.functional as F + +#from flash_attn import flash_attn_varlen_func + +from torch import nn + + +def pytorch_varlen_attention(q, k, v, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, dropout_p=0.0, softmax_scale=None, causal=False, deterministic=False): + """ + A PyTorch-based implementation of variable-length attention to replace flash_attn_varlen_func. + It processes each sequence in the batch individually. + """ + # Create an empty tensor to store the output. + output = torch.empty_like(q) + + # Iterate over each sequence in the batch. The batch size is the number of sequences. + for i in range(len(cu_seqlens_q) - 1): + # Determine the start and end indices for the current sequence. + start_q, end_q = cu_seqlens_q[i], cu_seqlens_q[i+1] + start_k, end_k = cu_seqlens_k[i], cu_seqlens_k[i+1] + + # Slice the q, k, and v tensors to get the data for the current sequence. + # The shape is (seq_len, heads, head_dim). + q_i = q[start_q:end_q] + k_i = k[start_k:end_k] + v_i = v[start_k:end_k] + + # Reshape for torch's scaled_dot_product_attention which expects (batch, heads, seq, dim). + # Here, we treat each sequence as a batch of 1. + q_i = q_i.permute(1, 0, 2).unsqueeze(0) # (1, heads, seq_len_q, head_dim) + k_i = k_i.permute(1, 0, 2).unsqueeze(0) # (1, heads, seq_len_k, head_dim) + v_i = v_i.permute(1, 0, 2).unsqueeze(0) # (1, heads, seq_len_k, head_dim) + + # Use PyTorch's built-in scaled dot-product attention. + output_i = F.scaled_dot_product_attention( + q_i, k_i, v_i, + dropout_p=dropout_p if not deterministic else 0.0, + is_causal=causal + ) + + # Reshape the output back to the original format (seq_len, heads, head_dim) + output_i = output_i.squeeze(0).permute(1, 0, 2) + + # Place the result for the current sequence into the main output tensor. + output[start_q:end_q] = output_i + + return output + + +class TorchAttention(nn.Module): + def tflops(self, args, kwargs, output) -> float: + assert len(args) == 0 or len(args) > 2, "query, key should both provided by args / kwargs" + q = kwargs.get("query") or args[0] + k = kwargs.get("key") or args[1] + b, h, sq, d = q.shape + b, h, sk, d = k.shape + return b * h * (4 * d * (sq / 1e6) * (sk / 1e6)) + + def forward(self, *args, **kwargs): + #return pytorch_varlen_attention(*args, **kwargs) + return F.scaled_dot_product_attention(*args, **kwargs) + + +class FlashAttentionVarlen(nn.Module): + def tflops(self, args, kwargs, output) -> float: + cu_seqlens_q = kwargs["cu_seqlens_q"] + cu_seqlens_k = kwargs["cu_seqlens_k"] + _, h, d = output.shape + seqlens_q = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]) / 1e6 + seqlens_k = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]) / 1e6 + return h * (4 * d * (seqlens_q * seqlens_k).sum()) + + def forward(self, *args, **kwargs): + kwargs["deterministic"] = torch.are_deterministic_algorithms_enabled() + try: + from flash_attn import flash_attn_varlen_func + return flash_attn_varlen_func(*args, **kwargs) + except ImportError: + return pytorch_varlen_attention(*args, **kwargs) diff --git a/modules/seedvr/src/models/dit/blocks/__init__.py b/modules/seedvr/src/models/dit/blocks/__init__.py new file mode 100644 index 000000000..3195b400a --- /dev/null +++ b/modules/seedvr/src/models/dit/blocks/__init__.py @@ -0,0 +1,25 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from .mmdit_window_block import MMWindowTransformerBlock + +dit_blocks = { + "mmdit_window": MMWindowTransformerBlock, +} + + +def get_block(block_type: str): + if block_type in dit_blocks: + return dit_blocks[block_type] + raise NotImplementedError(f"{block_type} is not supported") diff --git a/modules/seedvr/src/models/dit/blocks/mmdit_window_block.py b/modules/seedvr/src/models/dit/blocks/mmdit_window_block.py new file mode 100644 index 000000000..eb551aacc --- /dev/null +++ b/modules/seedvr/src/models/dit/blocks/mmdit_window_block.py @@ -0,0 +1,227 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Tuple, Union +import torch +from einops import rearrange +from torch import nn +from torch.nn import functional as F +from torch.nn.modules.utils import _triple +from ....common.half_precision_fixes import safe_pad_operation +from ....common.distributed.ops import gather_heads, gather_heads_scatter_seq, gather_seq_scatter_heads_qkv, scatter_heads +from ..attention import TorchAttention +from ..mlp import get_mlp +from ..mm import MMArg, MMModule +from ..modulation import ada_layer_type +from ..normalization import norm_layer_type +from ..rope import RotaryEmbedding3d + + +class MMWindowAttention(nn.Module): + def __init__( + self, + vid_dim: int, + txt_dim: int, + heads: int, + head_dim: int, + qk_bias: bool, + qk_rope: bool, + qk_norm: norm_layer_type, + qk_norm_eps: float, + window: Union[int, Tuple[int, int, int]], + window_method: str, + shared_qkv: bool, + ): + super().__init__() + dim = MMArg(vid_dim, txt_dim) + inner_dim = heads * head_dim + qkv_dim = inner_dim * 3 + + self.window = _triple(window) + self.window_method = window_method + assert all(map(lambda v: isinstance(v, int) and v >= 0, self.window)) + + self.head_dim = head_dim + self.proj_qkv = MMModule(nn.Linear, dim, qkv_dim, bias=qk_bias, shared_weights=shared_qkv) + self.proj_out = MMModule(nn.Linear, inner_dim, dim, shared_weights=shared_qkv) + self.norm_q = MMModule(qk_norm, dim=head_dim, eps=qk_norm_eps, elementwise_affine=True) + self.norm_k = MMModule(qk_norm, dim=head_dim, eps=qk_norm_eps, elementwise_affine=True) + self.rope = RotaryEmbedding3d(dim=head_dim // 2) if qk_rope else None + self.attn = TorchAttention() + + def forward( + self, + vid: torch.FloatTensor, # b T H W c + txt: torch.FloatTensor, # b L c + txt_mask: torch.BoolTensor, # b L + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + # Project q, k, v. + vid_qkv, txt_qkv = self.proj_qkv(vid, txt) + vid_qkv = gather_seq_scatter_heads_qkv(vid_qkv, seq_dim=2) + _, T, H, W, _ = vid_qkv.shape + _, L, _ = txt.shape + + if self.window_method == "win": + nt, nh, nw = self.window + tt, hh, ww = T // nt, H // nh, W // nw + elif self.window_method == "win_by_size": + tt, hh, ww = self.window + tt, hh, ww = ( + tt if tt > 0 else T, + hh if hh > 0 else H, + ww if ww > 0 else W, + ) + nt, nh, nw = T // tt, H // hh, W // ww + else: + raise NotImplementedError + + vid_qkv = rearrange(vid_qkv, "b T H W (o h d) -> o b h (T H W) d", o=3, d=self.head_dim) + txt_qkv = rearrange(txt_qkv, "b L (o h d) -> o b h L d", o=3, d=self.head_dim) + txt_qkv = scatter_heads(txt_qkv, dim=2) + + vid_q, vid_k, vid_v = vid_qkv.unbind() + txt_q, txt_k, txt_v = txt_qkv.unbind() + + vid_q, txt_q = self.norm_q(vid_q, txt_q) + vid_k, txt_k = self.norm_k(vid_k, txt_k) + + if self.rope: + vid_q, vid_k = self.rope(vid_q, vid_k, (T, H, W)) + + def vid_window(v): + return rearrange( + v, + "b h (nt tt nh hh nw ww) d -> b h (nt nh nw) (tt hh ww) d", + hh=hh, + ww=ww, + tt=tt, + nh=nh, + nw=nw, + nt=nt, + ) + + def txt_window(t): + return rearrange(t, "b h L d -> b h 1 L d").expand(-1, -1, nt * nh * nw, -1, -1) + + # Process video attention. + vid_msk = safe_pad_operation(txt_mask, (tt * hh * ww, 0), value=True) + vid_msk = rearrange(vid_msk, "b l -> b 1 1 1 l").expand(-1, 1, 1, tt * hh * ww, -1) + vid_out = self.attn( + vid_window(vid_q), + torch.cat([vid_window(vid_k), txt_window(txt_k)], dim=-2), + torch.cat([vid_window(vid_v), txt_window(txt_v)], dim=-2), + vid_msk, + ) + vid_out = rearrange( + vid_out, + "b h (nt nh nw) (tt hh ww) d -> b (nt tt) (nh hh) (nw ww) (h d)", + hh=hh, + ww=ww, + tt=tt, + nh=nh, + nw=nw, + ) + vid_out = gather_heads_scatter_seq(vid_out, head_dim=4, seq_dim=2) + + # Process text attention. + txt_msk = safe_pad_operation(txt_mask, (T * H * W, 0), value=True) + txt_msk = rearrange(txt_msk, "b l -> b 1 1 l").expand(-1, 1, L, -1) + txt_out = self.attn( + txt_q, + torch.cat([vid_k, txt_k], dim=-2), + torch.cat([vid_v, txt_v], dim=-2), + txt_msk, + ) + txt_out = rearrange(txt_out, "b h L d -> b L (h d)") + txt_out = gather_heads(txt_out, dim=2) + + # Project output. + vid_out, txt_out = self.proj_out(vid_out, txt_out) + return vid_out, txt_out + + +class MMWindowTransformerBlock(nn.Module): + def __init__( + self, + *, + vid_dim: int, + txt_dim: int, + emb_dim: int, + heads: int, + head_dim: int, + expand_ratio: int, + norm: norm_layer_type, + norm_eps: float, + ada: ada_layer_type, + qk_bias: bool, + qk_rope: bool, + qk_norm: norm_layer_type, + window: Union[int, Tuple[int, int, int]], + window_method: str, + shared_qkv: bool, + shared_mlp: bool, + mlp_type: str, + **kwargs, + ): + super().__init__() + dim = MMArg(vid_dim, txt_dim) + self.attn_norm = MMModule(norm, dim=dim, eps=norm_eps, elementwise_affine=False) + self.attn = MMWindowAttention( + vid_dim=vid_dim, + txt_dim=txt_dim, + heads=heads, + head_dim=head_dim, + qk_bias=qk_bias, + qk_rope=qk_rope, + qk_norm=qk_norm, + qk_norm_eps=norm_eps, + window=window, + window_method=window_method, + shared_qkv=shared_qkv, + ) + self.mlp_norm = MMModule(norm, dim=dim, eps=norm_eps, elementwise_affine=False) + self.mlp = MMModule( + get_mlp(mlp_type), + dim=dim, + expand_ratio=expand_ratio, + shared_weights=shared_mlp, + ) + self.ada = MMModule(ada, dim=dim, emb_dim=emb_dim, layers=["attn", "mlp"]) + + def forward( + self, + vid: torch.FloatTensor, + txt: torch.FloatTensor, + txt_mask: torch.BoolTensor, + emb: torch.FloatTensor, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + vid_attn, txt_attn = self.attn_norm(vid, txt) + vid_attn, txt_attn = self.ada(vid_attn, txt_attn, emb=emb, layer="attn", mode="in") + vid_attn, txt_attn = self.attn(vid_attn, txt_attn, txt_mask=txt_mask) + vid_attn, txt_attn = self.ada(vid_attn, txt_attn, emb=emb, layer="attn", mode="out") + vid_attn, txt_attn = (vid_attn + vid), (txt_attn + txt) + + vid_mlp, txt_mlp = self.mlp_norm(vid_attn, txt_attn) + vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, emb=emb, layer="mlp", mode="in") + vid_mlp, txt_mlp = self.mlp(vid_mlp, txt_mlp) + vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, emb=emb, layer="mlp", mode="out") + vid_mlp, txt_mlp = (vid_mlp + vid_attn), (txt_mlp + txt_attn) + + return vid_mlp, txt_mlp diff --git a/modules/seedvr/src/models/dit/embedding.py b/modules/seedvr/src/models/dit/embedding.py new file mode 100644 index 000000000..e972244f5 --- /dev/null +++ b/modules/seedvr/src/models/dit/embedding.py @@ -0,0 +1,62 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Optional, Union +import torch +from diffusers.models.embeddings import get_timestep_embedding +from torch import nn + + +def emb_add(emb1: torch.Tensor, emb2: Optional[torch.Tensor]): + return emb1 if emb2 is None else emb1 + emb2 + + +class TimeEmbedding(nn.Module): + def __init__( + self, + sinusoidal_dim: int, + hidden_dim: int, + output_dim: int, + ): + super().__init__() + self.sinusoidal_dim = sinusoidal_dim + self.proj_in = nn.Linear(sinusoidal_dim, hidden_dim) + self.proj_hid = nn.Linear(hidden_dim, hidden_dim) + self.proj_out = nn.Linear(hidden_dim, output_dim) + self.act = nn.SiLU() + + def forward( + self, + timestep: Union[int, float, torch.IntTensor, torch.FloatTensor], + device: torch.device, + dtype: torch.dtype, + ) -> torch.FloatTensor: + if not torch.is_tensor(timestep): + timestep = torch.tensor([timestep], device=device, dtype=dtype) + if timestep.ndim == 0: + timestep = timestep[None] + + emb = get_timestep_embedding( + timesteps=timestep, + embedding_dim=self.sinusoidal_dim, + flip_sin_to_cos=False, + downscale_freq_shift=0, + ) + emb = emb.to(dtype) + emb = self.proj_in(emb) + emb = self.act(emb) + emb = self.proj_hid(emb) + emb = self.act(emb) + emb = self.proj_out(emb) + return emb diff --git a/modules/seedvr/src/models/dit/mlp.py b/modules/seedvr/src/models/dit/mlp.py new file mode 100644 index 000000000..2d05cb021 --- /dev/null +++ b/modules/seedvr/src/models/dit/mlp.py @@ -0,0 +1,62 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Optional +import torch +import torch.nn.functional as F +from torch import nn + + +def get_mlp(mlp_type: Optional[str] = "normal"): + if mlp_type == "normal": + return MLP + elif mlp_type == "swiglu": + return SwiGLUMLP + + +class MLP(nn.Module): + def __init__( + self, + dim: int, + expand_ratio: int, + ): + super().__init__() + self.proj_in = nn.Linear(dim, dim * expand_ratio) + self.act = nn.GELU("tanh") + self.proj_out = nn.Linear(dim * expand_ratio, dim) + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + x = self.proj_in(x) + x = self.act(x) + x = self.proj_out(x) + return x + + +class SwiGLUMLP(nn.Module): + def __init__( + self, + dim: int, + expand_ratio: int, + multiple_of: int = 256, + ): + super().__init__() + hidden_dim = int(2 * dim * expand_ratio / 3) + hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of) + self.proj_in_gate = nn.Linear(dim, hidden_dim, bias=False) + self.proj_out = nn.Linear(hidden_dim, dim, bias=False) + self.proj_in = nn.Linear(dim, hidden_dim, bias=False) + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + x = self.proj_out(F.silu(self.proj_in_gate(x)) * self.proj_in(x)) + return x diff --git a/modules/seedvr/src/models/dit/mm.py b/modules/seedvr/src/models/dit/mm.py new file mode 100644 index 000000000..49be1f591 --- /dev/null +++ b/modules/seedvr/src/models/dit/mm.py @@ -0,0 +1,67 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Tuple +import torch +from torch import nn + + +@dataclass +class MMArg: + vid: Any + txt: Any + + +def get_args(key: str, args: List[Any]) -> List[Any]: + return [getattr(v, key) if isinstance(v, MMArg) else v for v in args] + + +def get_kwargs(key: str, kwargs: Dict[str, Any]) -> Dict[str, Any]: + return {k: getattr(v, key) if isinstance(v, MMArg) else v for k, v in kwargs.items()} + + +class MMModule(nn.Module): + def __init__( + self, + module: Callable[..., nn.Module], + *args, + shared_weights: bool = False, + **kwargs, + ): + super().__init__() + self.shared_weights = shared_weights + if self.shared_weights: + assert get_args("vid", args) == get_args("txt", args) + assert get_kwargs("vid", kwargs) == get_kwargs("txt", kwargs) + self.all = module(*get_args("vid", args), **get_kwargs("vid", kwargs)) + else: + self.vid = module(*get_args("vid", args), **get_kwargs("vid", kwargs)) + self.txt = module(*get_args("txt", args), **get_kwargs("txt", kwargs)) + + def forward( + self, + vid: torch.FloatTensor, + txt: torch.FloatTensor, + *args, + **kwargs, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + vid_module = self.vid if not self.shared_weights else self.all + txt_module = self.txt if not self.shared_weights else self.all + vid = vid_module(vid, *get_args("vid", args), **get_kwargs("vid", kwargs)) + txt = txt_module(txt, *get_args("txt", args), **get_kwargs("txt", kwargs)) + return vid, txt diff --git a/modules/seedvr/src/models/dit/modulation.py b/modules/seedvr/src/models/dit/modulation.py new file mode 100644 index 000000000..7a4975646 --- /dev/null +++ b/modules/seedvr/src/models/dit/modulation.py @@ -0,0 +1,97 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Callable, List, Optional +import torch +from einops import rearrange +from torch import nn + +from ...common.cache import Cache +from ...common.distributed.ops import slice_inputs + +# (dim: int, emb_dim: int) +ada_layer_type = Callable[[int, int], nn.Module] + + +def get_ada_layer(ada_layer: str) -> ada_layer_type: + if ada_layer == "single": + return AdaSingle + raise NotImplementedError(f"{ada_layer} is not supported") + + +def expand_dims(x: torch.Tensor, dim: int, ndim: int): + """ + Expand tensor "x" to "ndim" by adding empty dims at "dim". + Example: x is (b d), target ndim is 5, add dim at 1, return (b 1 1 1 d). + """ + shape = x.shape + shape = shape[:dim] + (1,) * (ndim - len(shape)) + shape[dim:] + return x.reshape(shape) + + +class AdaSingle(nn.Module): + def __init__( + self, + dim: int, + emb_dim: int, + layers: List[str], + ): + assert emb_dim == 6 * dim, "AdaSingle requires emb_dim == 6 * dim" + super().__init__() + self.dim = dim + self.emb_dim = emb_dim + self.layers = layers + for l in layers: + self.register_parameter(f"{l}_shift", nn.Parameter(torch.randn(dim) / dim**0.5)) + self.register_parameter(f"{l}_scale", nn.Parameter(torch.randn(dim) / dim**0.5 + 1)) + self.register_parameter(f"{l}_gate", nn.Parameter(torch.randn(dim) / dim**0.5)) + + def forward( + self, + hid: torch.FloatTensor, # b ... c + emb: torch.FloatTensor, # b d + layer: str, + mode: str, + cache: Cache = Cache(disable=True), + branch_tag: str = "", + hid_len: Optional[torch.LongTensor] = None, # b + ) -> torch.FloatTensor: + idx = self.layers.index(layer) + emb = rearrange(emb, "b (d l g) -> b d l g", l=len(self.layers), g=3)[..., idx, :] + emb = expand_dims(emb, 1, hid.ndim + 1) + + if hid_len is not None: + emb = cache( + f"emb_repeat_{idx}_{branch_tag}", + lambda: slice_inputs( + torch.cat([e.repeat(l, *([1] * e.ndim)) for e, l in zip(emb, hid_len)]), + dim=0, + ), + ) + + shiftA, scaleA, gateA = emb.unbind(-1) + shiftB, scaleB, gateB = ( + getattr(self, f"{layer}_shift"), + getattr(self, f"{layer}_scale"), + getattr(self, f"{layer}_gate"), + ) + + if mode == "in": + return hid.mul_(scaleA + scaleB).add_(shiftA + shiftB) + if mode == "out": + return hid.mul_(gateA + gateB) + raise NotImplementedError + + def extra_repr(self) -> str: + return f"dim={self.dim}, emb_dim={self.emb_dim}, layers={self.layers}" diff --git a/modules/seedvr/src/models/dit/na.py b/modules/seedvr/src/models/dit/na.py new file mode 100644 index 000000000..0dbd546c4 --- /dev/null +++ b/modules/seedvr/src/models/dit/na.py @@ -0,0 +1,241 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from itertools import chain +from typing import Callable, Dict, List, Tuple +import einops +import torch + + +def flatten( + hid: List[torch.FloatTensor], # List of (*** c) +) -> Tuple[ + torch.FloatTensor, # (L c) + torch.LongTensor, # (b n) +]: + assert len(hid) > 0 + shape = torch.stack([torch.tensor(x.shape[:-1], device=hid[0].device) for x in hid]) + hid = torch.cat([x.flatten(0, -2) for x in hid]) + return hid, shape + + +def unflatten( + hid: torch.FloatTensor, # (L c) or (L ... c) + hid_shape: torch.LongTensor, # (b n) +) -> List[torch.Tensor]: # List of (*** c) or (*** ... c) + hid_len = hid_shape.prod(-1) + hid = hid.split(hid_len.tolist()) + hid = [x.unflatten(0, s.tolist()) for x, s in zip(hid, hid_shape)] + return hid + + +def concat( + vid: torch.FloatTensor, # (VL ... c) + txt: torch.FloatTensor, # (TL ... c) + vid_len: torch.LongTensor, # (b) + txt_len: torch.LongTensor, # (b) +) -> torch.FloatTensor: # (L ... c) + vid = torch.split(vid, vid_len.tolist()) + txt = torch.split(txt, txt_len.tolist()) + return torch.cat(list(chain(*zip(vid, txt)))) + + +def concat_idx( + vid_len: torch.LongTensor, # (b) + txt_len: torch.LongTensor, # (b) +) -> Tuple[ + Callable, + Callable, +]: + device = vid_len.device + vid_idx = torch.arange(vid_len.sum(), device=device) + txt_idx = torch.arange(len(vid_idx), len(vid_idx) + txt_len.sum(), device=device) + tgt_idx = concat(vid_idx, txt_idx, vid_len, txt_len) + src_idx = torch.argsort(tgt_idx) + return ( + lambda vid, txt: torch.index_select(torch.cat([vid, txt]), 0, tgt_idx), + lambda all: torch.index_select(all, 0, src_idx).split([len(vid_idx), len(txt_idx)]), + ) + + +def unconcat( + all: torch.FloatTensor, # (L ... c) + vid_len: torch.LongTensor, # (b) + txt_len: torch.LongTensor, # (b) +) -> Tuple[ + torch.FloatTensor, # (VL ... c) + torch.FloatTensor, # (TL ... c) +]: + interleave_len = list(chain(*zip(vid_len.tolist(), txt_len.tolist()))) + all = all.split(interleave_len) + vid = torch.cat(all[0::2]) + txt = torch.cat(all[1::2]) + return vid, txt + + +def repeat_concat( + vid: torch.FloatTensor, # (VL ... c) + txt: torch.FloatTensor, # (TL ... c) + vid_len: torch.LongTensor, # (n*b) + txt_len: torch.LongTensor, # (b) + txt_repeat: List, # (n) +) -> torch.FloatTensor: # (L ... c) + vid = torch.split(vid, vid_len.tolist()) + txt = torch.split(txt, txt_len.tolist()) + txt = [[x] * n for x, n in zip(txt, txt_repeat)] + txt = list(chain(*txt)) + return torch.cat(list(chain(*zip(vid, txt)))) + + +def repeat_concat_idx( + vid_len: torch.LongTensor, # (n*b) + txt_len: torch.LongTensor, # (b) + txt_repeat: torch.LongTensor, # (n) +) -> Tuple[ + Callable, + Callable, +]: + device = vid_len.device + vid_idx = torch.arange(vid_len.sum(), device=device) + txt_idx = torch.arange(len(vid_idx), len(vid_idx) + txt_len.sum(), device=device) + txt_repeat_list = txt_repeat.tolist() + tgt_idx = repeat_concat(vid_idx, txt_idx, vid_len, txt_len, txt_repeat) + src_idx = torch.argsort(tgt_idx) + txt_idx_len = len(tgt_idx) - len(vid_idx) + repeat_txt_len = (txt_len * txt_repeat).tolist() + + def unconcat_coalesce(all): + """ + Un-concat vid & txt, and coalesce the repeated txt. + e.g. vid [0 1 2 3 4 5 6 7 8] -> 3 splits -> [0 1 2] [3 4 5] [6 7 8] + txt [9 10] + repeat_concat ==> [0 1 2 9 10 3 4 5 9 10 6 7 8 9 10] + 1. argsort re-index ==> [0 1 2 3 4 5 6 7 8 9 9 9 10 10 10] + split ==> vid_out [0 1 2 3 4 5 6 7 8] txt_out [9 9 9 10 10 10] + 2. reshape & mean for each sample to coalesce the repeated txt. + """ + vid_out, txt_out = all[src_idx].split([len(vid_idx), txt_idx_len]) + txt_out_coalesced = [] + for txt, repeat_time in zip(txt_out.split(repeat_txt_len), txt_repeat_list): + txt = txt.reshape(-1, repeat_time, *txt.shape[1:]).mean(1) + txt_out_coalesced.append(txt) + return vid_out, torch.cat(txt_out_coalesced) + + # Note: Backward of torch.index_select is non-deterministic when existing repeated index, + # the difference may cumulative like torch.repeat_interleave, so we use vanilla index here. + return ( + lambda vid, txt: torch.cat([vid, txt])[tgt_idx], + lambda all: unconcat_coalesce(all), + ) + + +def rearrange( + hid: torch.FloatTensor, # (L c) + hid_shape: torch.LongTensor, # (b n) + pattern: str, + **kwargs: Dict[str, int], +) -> Tuple[ + torch.FloatTensor, + torch.LongTensor, +]: + return flatten([einops.rearrange(h, pattern, **kwargs) for h in unflatten(hid, hid_shape)]) + + +def rearrange_idx( + hid_shape: torch.LongTensor, # (b n) + pattern: str, + **kwargs: Dict[str, int], +) -> Tuple[Callable, Callable, torch.LongTensor]: + hid_idx = torch.arange(hid_shape.prod(-1).sum(), device=hid_shape.device).unsqueeze(-1) + tgt_idx, tgt_shape = rearrange(hid_idx, hid_shape, pattern, **kwargs) + tgt_idx = tgt_idx.squeeze(-1) + src_idx = torch.argsort(tgt_idx) + return ( + lambda hid: torch.index_select(hid, 0, tgt_idx), + lambda hid: torch.index_select(hid, 0, src_idx), + tgt_shape, + ) + + +def repeat( + hid: torch.FloatTensor, # (L c) + hid_shape: torch.LongTensor, # (b n) + pattern: str, + **kwargs: Dict[str, torch.LongTensor], # (b) +) -> Tuple[ + torch.FloatTensor, + torch.LongTensor, +]: + hid = unflatten(hid, hid_shape) + kwargs = [{k: v[i].item() for k, v in kwargs.items()} for i in range(len(hid))] + return flatten([einops.repeat(h, pattern, **a) for h, a in zip(hid, kwargs)]) + + +def pack( + samples: List[torch.Tensor], # List of (h w c). +) -> Tuple[ + List[torch.Tensor], # groups [(b1 h1 w1 c1), (b2 h2 w2 c2)] + List[List[int]], # reversal indices. +]: + batches = {} + indices = {} + for i, sample in enumerate(samples): + shape = sample.shape + batches[shape] = batches.get(shape, []) + indices[shape] = indices.get(shape, []) + batches[shape].append(sample) + indices[shape].append(i) + + batches = list(map(torch.stack, batches.values())) + indices = list(indices.values()) + return batches, indices + + +def unpack( + batches: List[torch.Tensor], + indices: List[List[int]], +) -> List[torch.Tensor]: + samples = [None] * (max(chain(*indices)) + 1) + for batch, index in zip(batches, indices): + for sample, i in zip(batch.unbind(), index): + samples[i] = sample + return samples + + +def window( + hid: torch.FloatTensor, # (L c) + hid_shape: torch.LongTensor, # (b n) + window_fn: Callable[[torch.Tensor], List[torch.Tensor]], +): + hid = unflatten(hid, hid_shape) + hid = list(map(window_fn, hid)) + hid_windows = torch.tensor(list(map(len, hid)), device=hid_shape.device) + hid, hid_shape = flatten(list(chain(*hid))) + return hid, hid_shape, hid_windows + + +def window_idx( + hid_shape: torch.LongTensor, # (b n) + window_fn: Callable[[torch.Tensor], List[torch.Tensor]], +): + hid_idx = torch.arange(hid_shape.prod(-1).sum(), device=hid_shape.device).unsqueeze(-1) + tgt_idx, tgt_shape, tgt_windows = window(hid_idx, hid_shape, window_fn) + tgt_idx = tgt_idx.squeeze(-1) + src_idx = torch.argsort(tgt_idx) + return ( + lambda hid: torch.index_select(hid, 0, tgt_idx), + lambda hid: torch.index_select(hid, 0, src_idx), + tgt_shape, + tgt_windows, + ) diff --git a/modules/seedvr/src/models/dit/nablocks/__init__.py b/modules/seedvr/src/models/dit/nablocks/__init__.py new file mode 100644 index 000000000..afa206db1 --- /dev/null +++ b/modules/seedvr/src/models/dit/nablocks/__init__.py @@ -0,0 +1,25 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from .mmsr_block import NaMMSRTransformerBlock + +nadit_blocks = { + "mmdit_sr": NaMMSRTransformerBlock, +} + + +def get_nablock(block_type: str): + if block_type in nadit_blocks: + return nadit_blocks[block_type] + raise NotImplementedError(f"{block_type} is not supported") diff --git a/modules/seedvr/src/models/dit/nablocks/mmsr_block.py b/modules/seedvr/src/models/dit/nablocks/mmsr_block.py new file mode 100644 index 000000000..48e548262 --- /dev/null +++ b/modules/seedvr/src/models/dit/nablocks/mmsr_block.py @@ -0,0 +1,248 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Tuple, Union +import torch +from einops import rearrange +from torch.nn import functional as F + +# from ..cache import Cache +from ....common.cache import Cache +from ....common.distributed.ops import gather_heads_scatter_seq, gather_seq_scatter_heads_qkv + +from .. import na +from ..attention import FlashAttentionVarlen +from ..blocks.mmdit_window_block import MMWindowAttention, MMWindowTransformerBlock +from ..mm import MMArg +from ..modulation import ada_layer_type +from ..normalization import norm_layer_type +from ..rope import NaRotaryEmbedding3d +from ..window import get_window_op +from ....common.half_precision_fixes import safe_pad_operation + +class NaSwinAttention(MMWindowAttention): + def __init__( + self, + vid_dim: int, + txt_dim: int, + heads: int, + head_dim: int, + qk_bias: bool, + qk_rope: bool, + qk_norm: norm_layer_type, + qk_norm_eps: float, + window: Union[int, Tuple[int, int, int]], + window_method: str, + shared_qkv: bool, + **kwargs, + ): + super().__init__( + vid_dim=vid_dim, + txt_dim=txt_dim, + heads=heads, + head_dim=head_dim, + qk_bias=qk_bias, + qk_rope=qk_rope, + qk_norm=qk_norm, + qk_norm_eps=qk_norm_eps, + window=window, + window_method=window_method, + shared_qkv=shared_qkv, + ) + self.rope = NaRotaryEmbedding3d(dim=head_dim // 2) if qk_rope else None + self.attn = FlashAttentionVarlen() + self.window_op = get_window_op(window_method) + + def forward( + self, + vid: torch.FloatTensor, # l c + txt: torch.FloatTensor, # l c + vid_shape: torch.LongTensor, # b 3 + txt_shape: torch.LongTensor, # b 1 + cache: Cache, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + + vid_qkv, txt_qkv = self.proj_qkv(vid, txt) + vid_qkv = gather_seq_scatter_heads_qkv( + vid_qkv, + seq_dim=0, + qkv_shape=vid_shape, + cache=cache.namespace("vid"), + ) + txt_qkv = gather_seq_scatter_heads_qkv( + txt_qkv, + seq_dim=0, + qkv_shape=txt_shape, + cache=cache.namespace("txt"), + ) + + # re-org the input seq for window attn + cache_win = cache.namespace(f"{self.window_method}_{self.window}_sd3") + + def make_window(x: torch.Tensor): + t, h, w, _ = x.shape + window_slices = self.window_op((t, h, w), self.window) + return [x[st, sh, sw] for (st, sh, sw) in window_slices] + + window_partition, window_reverse, window_shape, window_count = cache_win( + "win_transform", + lambda: na.window_idx(vid_shape, make_window), + ) + vid_qkv_win = window_partition(vid_qkv) + + vid_qkv_win = rearrange(vid_qkv_win, "l (o h d) -> l o h d", o=3, d=self.head_dim) + txt_qkv = rearrange(txt_qkv, "l (o h d) -> l o h d", o=3, d=self.head_dim) + + vid_q, vid_k, vid_v = vid_qkv_win.unbind(1) + txt_q, txt_k, txt_v = txt_qkv.unbind(1) + + vid_q, txt_q = self.norm_q(vid_q, txt_q) + vid_k, txt_k = self.norm_k(vid_k, txt_k) + + txt_len = cache("txt_len", lambda: txt_shape.prod(-1)) + + vid_len_win = cache_win("vid_len", lambda: window_shape.prod(-1)) + txt_len_win = cache_win("txt_len", lambda: txt_len.repeat_interleave(window_count)) + all_len_win = cache_win("all_len", lambda: vid_len_win + txt_len_win) + concat_win, unconcat_win = cache_win( + "mm_pnp", lambda: na.repeat_concat_idx(vid_len_win, txt_len, window_count) + ) + + # window rope + if self.rope: + vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win) + + out = self.attn( + q=concat_win(vid_q, txt_q).bfloat16(), + k=concat_win(vid_k, txt_k).bfloat16(), + v=concat_win(vid_v, txt_v).bfloat16(), + cu_seqlens_q=cache_win( + "vid_seqlens_q", lambda: safe_pad_operation(all_len_win.cumsum(0), (1, 0)).int() + ), + cu_seqlens_k=cache_win( + "vid_seqlens_k", lambda: safe_pad_operation(all_len_win.cumsum(0), (1, 0)).int() + ), + max_seqlen_q=cache_win("vid_max_seqlen_q", lambda: all_len_win.max().item()), + max_seqlen_k=cache_win("vid_max_seqlen_k", lambda: all_len_win.max().item()), + ).type_as(vid_q) + + # text pooling + vid_out, txt_out = unconcat_win(out) + + vid_out = rearrange(vid_out, "l h d -> l (h d)") + txt_out = rearrange(txt_out, "l h d -> l (h d)") + vid_out = window_reverse(vid_out) + + vid_out = gather_heads_scatter_seq(vid_out, head_dim=1, seq_dim=0) + txt_out = gather_heads_scatter_seq(txt_out, head_dim=1, seq_dim=0) + + vid_out, txt_out = self.proj_out(vid_out, txt_out) + + return vid_out, txt_out + + +class NaMMSRTransformerBlock(MMWindowTransformerBlock): + def __init__( + self, + *, + vid_dim: int, + txt_dim: int, + emb_dim: int, + heads: int, + head_dim: int, + expand_ratio: int, + norm: norm_layer_type, + norm_eps: float, + ada: ada_layer_type, + qk_bias: bool, + qk_rope: bool, + qk_norm: norm_layer_type, + shared_qkv: bool, + shared_mlp: bool, + mlp_type: str, + **kwargs, + ): + super().__init__( + vid_dim=vid_dim, + txt_dim=txt_dim, + emb_dim=emb_dim, + heads=heads, + head_dim=head_dim, + expand_ratio=expand_ratio, + norm=norm, + norm_eps=norm_eps, + ada=ada, + qk_bias=qk_bias, + qk_rope=qk_rope, + qk_norm=qk_norm, + shared_qkv=shared_qkv, + shared_mlp=shared_mlp, + mlp_type=mlp_type, + **kwargs, + ) + + self.attn = NaSwinAttention( + vid_dim=vid_dim, + txt_dim=txt_dim, + heads=heads, + head_dim=head_dim, + qk_bias=qk_bias, + qk_rope=qk_rope, + qk_norm=qk_norm, + qk_norm_eps=norm_eps, + shared_qkv=shared_qkv, + **kwargs, + ) + + def forward( + self, + vid: torch.FloatTensor, # l c + txt: torch.FloatTensor, # l c + vid_shape: torch.LongTensor, # b 3 + txt_shape: torch.LongTensor, # b 1 + emb: torch.FloatTensor, + cache: Cache, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + torch.LongTensor, + torch.LongTensor, + ]: + hid_len = MMArg( + cache("vid_len", lambda: vid_shape.prod(-1)), + cache("txt_len", lambda: txt_shape.prod(-1)), + ) + ada_kwargs = { + "emb": emb, + "hid_len": hid_len, + "cache": cache, + "branch_tag": MMArg("vid", "txt"), + } + + vid_attn, txt_attn = self.attn_norm(vid, txt) + vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="in", **ada_kwargs) + vid_attn, txt_attn = self.attn(vid_attn, txt_attn, vid_shape, txt_shape, cache) + vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="out", **ada_kwargs) + vid_attn, txt_attn = (vid_attn + vid), (txt_attn + txt) + + vid_mlp, txt_mlp = self.mlp_norm(vid_attn, txt_attn) + vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="in", **ada_kwargs) + vid_mlp, txt_mlp = self.mlp(vid_mlp, txt_mlp) + vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="out", **ada_kwargs) + vid_mlp, txt_mlp = (vid_mlp + vid_attn), (txt_mlp + txt_attn) + + return vid_mlp, txt_mlp, vid_shape, txt_shape diff --git a/modules/seedvr/src/models/dit/nadit.py b/modules/seedvr/src/models/dit/nadit.py new file mode 100644 index 000000000..a67695206 --- /dev/null +++ b/modules/seedvr/src/models/dit/nadit.py @@ -0,0 +1,350 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from dataclasses import dataclass +from typing import Optional, Tuple, Union, Callable +import torch +from torch import nn + +from ...common.cache import Cache +from ...common.distributed.ops import slice_inputs + +from . import na +from .embedding import TimeEmbedding +from .modulation import get_ada_layer +from .nablocks import get_nablock +from .normalization import get_norm_layer +from .patch import NaPatchIn, NaPatchOut + +# Fake func, no checkpointing is required for inference +def gradient_checkpointing(module: Union[Callable, nn.Module], *args, enabled: bool, **kwargs): + return module(*args, **kwargs) + +@dataclass +class NaDiTOutput: + vid_sample: torch.Tensor + + +class NaDiT(nn.Module): + """ + Native Resolution Diffusion Transformer (NaDiT) + """ + + gradient_checkpointing = False + + def __init__( + self, + vid_in_channels: int, + vid_out_channels: int, + vid_dim: int, + txt_in_dim: Optional[int], + txt_dim: Optional[int], + emb_dim: int, + heads: int, + head_dim: int, + expand_ratio: int, + norm: Optional[str], + norm_eps: float, + ada: str, + qk_bias: bool, + qk_rope: bool, + qk_norm: Optional[str], + patch_size: Union[int, Tuple[int, int, int]], + num_layers: int, + block_type: Union[str, Tuple[str]], + shared_qkv: bool = False, + shared_mlp: bool = False, + mlp_type: str = "normal", + window: Optional[Tuple] = None, + window_method: Optional[Tuple[str]] = None, + temporal_window_size: int = None, + temporal_shifted: bool = False, + **kwargs, + ): + ada = get_ada_layer(ada) + norm = get_norm_layer(norm) + qk_norm = get_norm_layer(qk_norm) + if isinstance(block_type, str): + block_type = [block_type] * num_layers + elif len(block_type) != num_layers: + raise ValueError("The ``block_type`` list should equal to ``num_layers``.") + super().__init__() + self.vid_in = NaPatchIn( + in_channels=vid_in_channels, + patch_size=patch_size, + dim=vid_dim, + ) + self.txt_in = ( + nn.Linear(txt_in_dim, txt_dim) + if txt_in_dim and txt_in_dim != txt_dim + else nn.Identity() + ) + self.emb_in = TimeEmbedding( + sinusoidal_dim=256, + hidden_dim=max(vid_dim, txt_dim), + output_dim=emb_dim, + ) + + if window is None or isinstance(window[0], int): + window = [window] * num_layers + if window_method is None or isinstance(window_method, str): + window_method = [window_method] * num_layers + if temporal_window_size is None or isinstance(temporal_window_size, int): + temporal_window_size = [temporal_window_size] * num_layers + if temporal_shifted is None or isinstance(temporal_shifted, bool): + temporal_shifted = [temporal_shifted] * num_layers + + self.blocks = nn.ModuleList( + [ + get_nablock(block_type[i])( + vid_dim=vid_dim, + txt_dim=txt_dim, + emb_dim=emb_dim, + heads=heads, + head_dim=head_dim, + expand_ratio=expand_ratio, + norm=norm, + norm_eps=norm_eps, + ada=ada, + qk_bias=qk_bias, + qk_rope=qk_rope, + qk_norm=qk_norm, + shared_qkv=shared_qkv, + shared_mlp=shared_mlp, + mlp_type=mlp_type, + window=window[i], + window_method=window_method[i], + temporal_window_size=temporal_window_size[i], + temporal_shifted=temporal_shifted[i], + **kwargs, + ) + for i in range(num_layers) + ] + ) + self.vid_out = NaPatchOut( + out_channels=vid_out_channels, + patch_size=patch_size, + dim=vid_dim, + ) + + self.need_txt_repeat = block_type[0] in [ + "mmdit_stwin", + "mmdit_stwin_spatial", + "mmdit_stwin_3d_spatial", + ] + + def set_gradient_checkpointing(self, enable: bool): + self.gradient_checkpointing = enable + + def forward( + self, + vid: torch.FloatTensor, # l c + txt: torch.FloatTensor, # l c + vid_shape: torch.LongTensor, # b 3 + txt_shape: torch.LongTensor, # b 1 + timestep: Union[int, float, torch.IntTensor, torch.FloatTensor], # b + disable_cache: bool = True, # for test + ): + # Text input. + if txt_shape.size(-1) == 1 and self.need_txt_repeat: + txt, txt_shape = na.repeat(txt, txt_shape, "l c -> t l c", t=vid_shape[:, 0]) + # slice vid after patching in when using sequence parallelism + txt = slice_inputs(txt, dim=0) + txt = self.txt_in(txt) + + # Video input. + # Sequence parallel slicing is done inside patching class. + vid, vid_shape = self.vid_in(vid, vid_shape) + + # Embedding input. + emb = self.emb_in(timestep, device=vid.device, dtype=vid.dtype) + + # Body + cache = Cache(disable=disable_cache) + for i, block in enumerate(self.blocks): + vid, txt, vid_shape, txt_shape = gradient_checkpointing( + enabled=(self.gradient_checkpointing and self.training), + module=block, + vid=vid, + txt=txt, + vid_shape=vid_shape, + txt_shape=txt_shape, + emb=emb, + cache=cache, + ) + + vid, vid_shape = self.vid_out(vid, vid_shape, cache) + return NaDiTOutput(vid_sample=vid) + + +class NaDiTUpscaler(nn.Module): + """ + Native Resolution Diffusion Transformer (NaDiT) + """ + + gradient_checkpointing = False + + def __init__( + self, + vid_in_channels: int, + vid_out_channels: int, + vid_dim: int, + txt_in_dim: Optional[int], + txt_dim: Optional[int], + emb_dim: int, + heads: int, + head_dim: int, + expand_ratio: int, + norm: Optional[str], + norm_eps: float, + ada: str, + qk_bias: bool, + qk_rope: bool, + qk_norm: Optional[str], + patch_size: Union[int, Tuple[int, int, int]], + num_layers: int, + block_type: Union[str, Tuple[str]], + shared_qkv: bool = False, + shared_mlp: bool = False, + mlp_type: str = "normal", + window: Optional[Tuple] = None, + window_method: Optional[Tuple[str]] = None, + temporal_window_size: int = None, + temporal_shifted: bool = False, + **kwargs, + ): + ada = get_ada_layer(ada) + norm = get_norm_layer(norm) + qk_norm = get_norm_layer(qk_norm) + if isinstance(block_type, str): + block_type = [block_type] * num_layers + elif len(block_type) != num_layers: + raise ValueError("The ``block_type`` list should equal to ``num_layers``.") + super().__init__() + self.vid_in = NaPatchIn( + in_channels=vid_in_channels, + patch_size=patch_size, + dim=vid_dim, + ) + self.txt_in = ( + nn.Linear(txt_in_dim, txt_dim) + if txt_in_dim and txt_in_dim != txt_dim + else nn.Identity() + ) + self.emb_in = TimeEmbedding( + sinusoidal_dim=256, + hidden_dim=max(vid_dim, txt_dim), + output_dim=emb_dim, + ) + + self.emb_scale = TimeEmbedding( + sinusoidal_dim=256, + hidden_dim=max(vid_dim, txt_dim), + output_dim=emb_dim, + ) + + if window is None or isinstance(window[0], int): + window = [window] * num_layers + if window_method is None or isinstance(window_method, str): + window_method = [window_method] * num_layers + if temporal_window_size is None or isinstance(temporal_window_size, int): + temporal_window_size = [temporal_window_size] * num_layers + if temporal_shifted is None or isinstance(temporal_shifted, bool): + temporal_shifted = [temporal_shifted] * num_layers + + self.blocks = nn.ModuleList( + [ + get_nablock(block_type[i])( + vid_dim=vid_dim, + txt_dim=txt_dim, + emb_dim=emb_dim, + heads=heads, + head_dim=head_dim, + expand_ratio=expand_ratio, + norm=norm, + norm_eps=norm_eps, + ada=ada, + qk_bias=qk_bias, + qk_rope=qk_rope, + qk_norm=qk_norm, + shared_qkv=shared_qkv, + shared_mlp=shared_mlp, + mlp_type=mlp_type, + window=window[i], + window_method=window_method[i], + temporal_window_size=temporal_window_size[i], + temporal_shifted=temporal_shifted[i], + **kwargs, + ) + for i in range(num_layers) + ] + ) + self.vid_out = NaPatchOut( + out_channels=vid_out_channels, + patch_size=patch_size, + dim=vid_dim, + ) + + self.need_txt_repeat = block_type[0] in [ + "mmdit_stwin", + "mmdit_stwin_spatial", + "mmdit_stwin_3d_spatial", + ] + + def set_gradient_checkpointing(self, enable: bool): + self.gradient_checkpointing = enable + + def forward( + self, + vid: torch.FloatTensor, # l c + txt: torch.FloatTensor, # l c + vid_shape: torch.LongTensor, # b 3 + txt_shape: torch.LongTensor, # b 1 + timestep: Union[int, float, torch.IntTensor, torch.FloatTensor], # b + downscale: Union[int, float, torch.IntTensor, torch.FloatTensor], # b + disable_cache: bool = False, # for test + ): + + # Text input. + if txt_shape.size(-1) == 1 and self.need_txt_repeat: + txt, txt_shape = na.repeat(txt, txt_shape, "l c -> t l c", t=vid_shape[:, 0]) + # slice vid after patching in when using sequence parallelism + txt = slice_inputs(txt, dim=0) + txt = self.txt_in(txt) + + # Video input. + # Sequence parallel slicing is done inside patching class. + vid, vid_shape = self.vid_in(vid, vid_shape) + + # Embedding input. + emb = self.emb_in(timestep, device=vid.device, dtype=vid.dtype) + emb_scale = self.emb_scale(downscale, device=vid.device, dtype=vid.dtype) + emb = emb + emb_scale + + # Body + cache = Cache(disable=disable_cache) + for i, block in enumerate(self.blocks): + vid, txt, vid_shape, txt_shape = gradient_checkpointing( + enabled=(self.gradient_checkpointing and self.training), + module=block, + vid=vid, + txt=txt, + vid_shape=vid_shape, + txt_shape=txt_shape, + emb=emb, + cache=cache, + ) + + vid, vid_shape = self.vid_out(vid, vid_shape, cache) + return NaDiTOutput(vid_sample=vid) diff --git a/modules/seedvr/src/models/dit/normalization.py b/modules/seedvr/src/models/dit/normalization.py new file mode 100644 index 000000000..faa2d0d88 --- /dev/null +++ b/modules/seedvr/src/models/dit/normalization.py @@ -0,0 +1,131 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Callable, Optional +from diffusers.models.normalization import RMSNorm +from torch import nn +import torch +import torch.nn.functional as F +import numbers +from torch.nn.parameter import Parameter +from torch.nn import init + +# (dim: int, eps: float, elementwise_affine: bool) +norm_layer_type = Callable[[int, float, bool], nn.Module] + + +class CustomLayerNorm(nn.Module): + """ + Custom LayerNorm implementation to replace Apex FusedLayerNorm + """ + def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True): + super(CustomLayerNorm, self).__init__() + + if isinstance(normalized_shape, numbers.Integral): + normalized_shape = (normalized_shape,) + self.normalized_shape = torch.Size(normalized_shape) + self.eps = eps + self.elementwise_affine = elementwise_affine + + if self.elementwise_affine: + self.weight = Parameter(torch.Tensor(*normalized_shape)) + self.bias = Parameter(torch.Tensor(*normalized_shape)) + else: + self.register_parameter('weight', None) + self.register_parameter('bias', None) + self.reset_parameters() + + def reset_parameters(self): + if self.elementwise_affine: + init.ones_(self.weight) + init.zeros_(self.bias) + + def forward(self, input): + return F.layer_norm( + input, self.normalized_shape, self.weight, self.bias, self.eps) + + +class CustomRMSNorm(nn.Module): + """ + Custom RMSNorm implementation to replace Apex FusedRMSNorm + """ + def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True): + super(CustomRMSNorm, self).__init__() + + if isinstance(normalized_shape, numbers.Integral): + normalized_shape = (normalized_shape,) + self.normalized_shape = torch.Size(normalized_shape) + self.eps = eps + self.elementwise_affine = elementwise_affine + + if self.elementwise_affine: + self.weight = Parameter(torch.ones(*normalized_shape)) + else: + self.register_parameter('weight', None) + + def forward(self, input): + # RMS normalization: x / sqrt(mean(x^2) + eps) * weight + dims = tuple(range(-len(self.normalized_shape), 0)) + + # Calculate RMS: sqrt(mean(x^2)) + variance = input.pow(2).mean(dim=dims, keepdim=True) + rms = torch.sqrt(variance + self.eps) + + # Normalize + normalized = input / rms + + if self.elementwise_affine: + return normalized * self.weight + return normalized + + +def get_norm_layer(norm_type: Optional[str]) -> norm_layer_type: + + def _norm_layer(dim: int, eps: float, elementwise_affine: bool): + if norm_type is None: + return nn.Identity() + + if norm_type == "layer": + return nn.LayerNorm( + normalized_shape=dim, + eps=eps, + elementwise_affine=elementwise_affine, + ) + + if norm_type == "rms": + return RMSNorm( + dim=dim, + eps=eps, + elementwise_affine=elementwise_affine, + ) + + if norm_type == "fusedln": + # Use custom LayerNorm instead of Apex FusedLayerNorm + return CustomLayerNorm( + normalized_shape=dim, + elementwise_affine=elementwise_affine, + eps=eps, + ) + + if norm_type == "fusedrms": + # Use custom RMSNorm instead of Apex FusedRMSNorm + return CustomRMSNorm( + normalized_shape=dim, + elementwise_affine=elementwise_affine, + eps=eps, + ) + + raise NotImplementedError(f"{norm_type} is not supported") + + return _norm_layer diff --git a/modules/seedvr/src/models/dit/patch.py b/modules/seedvr/src/models/dit/patch.py new file mode 100644 index 000000000..570e3ae40 --- /dev/null +++ b/modules/seedvr/src/models/dit/patch.py @@ -0,0 +1,112 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Tuple, Union +import torch +from einops import rearrange +from torch import nn +from torch.nn.modules.utils import _triple + +from ...common.cache import Cache +from ...common.distributed.ops import gather_outputs, slice_inputs + +from . import na + + +class PatchIn(nn.Module): + def __init__( + self, + in_channels: int, + patch_size: Union[int, Tuple[int, int, int]], + dim: int, + ): + super().__init__() + t, h, w = _triple(patch_size) + self.patch_size = t, h, w + self.proj = nn.Linear(in_channels * t * h * w, dim) + + def forward( + self, + vid: torch.Tensor, + ) -> torch.Tensor: + t, h, w = self.patch_size + vid = rearrange(vid, "b c (T t) (H h) (W w) -> b T H W (t h w c)", t=t, h=h, w=w) + vid = self.proj(vid) + return vid + + +class PatchOut(nn.Module): + def __init__( + self, + out_channels: int, + patch_size: Union[int, Tuple[int, int, int]], + dim: int, + ): + super().__init__() + t, h, w = _triple(patch_size) + self.patch_size = t, h, w + self.proj = nn.Linear(dim, out_channels * t * h * w) + + def forward( + self, + vid: torch.Tensor, + ) -> torch.Tensor: + t, h, w = self.patch_size + vid = self.proj(vid) + vid = rearrange(vid, "b T H W (t h w c) -> b c (T t) (H h) (W w)", t=t, h=h, w=w) + return vid + + +class NaPatchIn(PatchIn): + def forward( + self, + vid: torch.Tensor, # l c + vid_shape: torch.LongTensor, + ) -> torch.Tensor: + t, h, w = self.patch_size + if not (t == h == w == 1): + vid, vid_shape = na.rearrange( + vid, vid_shape, "(T t) (H h) (W w) c -> T H W (t h w c)", t=t, h=h, w=w + ) + # slice vid after patching in when using sequence parallelism + vid = slice_inputs(vid, dim=0) + vid = self.proj(vid) + return vid, vid_shape + + +class NaPatchOut(PatchOut): + def forward( + self, + vid: torch.FloatTensor, # l c + vid_shape: torch.LongTensor, + cache: Cache = Cache(disable=True), + ) -> Tuple[ + torch.FloatTensor, + torch.LongTensor, + ]: + t, h, w = self.patch_size + vid = self.proj(vid) + # gather vid before patching out when enabling sequence parallelism + vid = gather_outputs( + vid, + gather_dim=0, + padding_dim=0, + unpad_shape=vid_shape, + cache=cache.namespace("vid"), + ) + if not (t == h == w == 1): + vid, vid_shape = na.rearrange( + vid, vid_shape, "T H W (t h w c) -> (T t) (H h) (W w) c", t=t, h=h, w=w + ) + return vid, vid_shape diff --git a/modules/seedvr/src/models/dit/rope.py b/modules/seedvr/src/models/dit/rope.py new file mode 100644 index 000000000..35b91ea8b --- /dev/null +++ b/modules/seedvr/src/models/dit/rope.py @@ -0,0 +1,101 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from functools import lru_cache +from typing import Tuple +import torch +from einops import rearrange +from torch import nn +from ...common.cache import Cache +from ....rotary_embedding import RotaryEmbedding + + +class RotaryEmbeddingBase(nn.Module): + def __init__(self, dim: int, rope_dim: int): + super().__init__() + self.rope = RotaryEmbedding( + dim=dim // rope_dim, + freqs_for="pixel", + max_freq=256, + ) + # 1. Set model.requires_grad_(True) after model creation will make + # the `requires_grad=False` for rope freqs no longer hold. + # 2. Even if we don't set requires_grad_(True) explicitly, + # FSDP is not memory efficient when handling fsdp_wrap + # with mixed requires_grad=True/False. + # With above consideration, it is easier just remove the freqs + # out of nn.Parameters when `learned_freq=False` + freqs = self.rope.freqs + del self.rope.freqs + self.rope.register_buffer("freqs", freqs.data) + + @lru_cache(maxsize=128) + def get_axial_freqs(self, *dims): + return self.rope.get_axial_freqs(*dims) + + +class RotaryEmbedding3d(RotaryEmbeddingBase): + def __init__(self, dim: int): + super().__init__(dim, rope_dim=3) + + def forward( + self, + q: torch.FloatTensor, # b h l d + k: torch.FloatTensor, # b h l d + size: Tuple[int, int, int], + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + T, H, W = size + freqs = self.get_axial_freqs(T, H, W) + q = rearrange(q, "b h (T H W) d -> b h T H W d", T=T, H=H, W=W) + k = rearrange(k, "b h (T H W) d -> b h T H W d", T=T, H=H, W=W) + q = apply_rotary_emb(freqs, q) + k = apply_rotary_emb(freqs, k) + q = rearrange(q, "b h T H W d -> b h (T H W) d") + k = rearrange(k, "b h T H W d -> b h (T H W) d") + return q, k + + +class NaRotaryEmbedding3d(RotaryEmbedding3d): + def forward( + self, + q: torch.FloatTensor, # L h d + k: torch.FloatTensor, # L h d + shape: torch.LongTensor, + cache: Cache, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + freqs = cache("rope_freqs_3d", lambda: self.get_freqs(shape)) + freqs = freqs.to(device=q.device, dtype=q.dtype) + q = rearrange(q, "L h d -> h L d") + k = rearrange(k, "L h d -> h L d") + q = apply_rotary_emb(freqs, q.float()).to(q.dtype) + k = apply_rotary_emb(freqs, k.float()).to(k.dtype) + q = rearrange(q, "h L d -> L h d") + k = rearrange(k, "h L d -> L h d") + return q, k + + def get_freqs( + self, + shape: torch.LongTensor, + ) -> torch.Tensor: + freq_list = [] + for f, h, w in shape.tolist(): + freqs = self.get_axial_freqs(f, h, w) + freq_list.append(freqs.view(-1, freqs.size(-1))) + return torch.cat(freq_list, dim=0) diff --git a/modules/seedvr/src/models/dit/window.py b/modules/seedvr/src/models/dit/window.py new file mode 100644 index 000000000..f77ea57d7 --- /dev/null +++ b/modules/seedvr/src/models/dit/window.py @@ -0,0 +1,83 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from math import ceil +from typing import Tuple +import math + +def get_window_op(name: str): + if name == "720pwin_by_size_bysize": + return make_720Pwindows_bysize + if name == "720pswin_by_size_bysize": + return make_shifted_720Pwindows_bysize + raise ValueError(f"Unknown windowing method: {name}") + + +# -------------------------------- Windowing -------------------------------- # +def make_720Pwindows_bysize(size: Tuple[int, int, int], num_windows: Tuple[int, int, int]): + t, h, w = size + resized_nt, resized_nh, resized_nw = num_windows + #cal windows under 720p + scale = math.sqrt((45 * 80) / (h * w)) + resized_h, resized_w = round(h * scale), round(w * scale) + wh, ww = ceil(resized_h / resized_nh), ceil(resized_w / resized_nw) # window size. + wt = ceil(min(t, 30) / resized_nt) # window size. + nt, nh, nw = ceil(t / wt), ceil(h / wh), ceil(w / ww) # window size. + return [ + ( + slice(it * wt, min((it + 1) * wt, t)), + slice(ih * wh, min((ih + 1) * wh, h)), + slice(iw * ww, min((iw + 1) * ww, w)), + ) + for iw in range(nw) + if min((iw + 1) * ww, w) > iw * ww + for ih in range(nh) + if min((ih + 1) * wh, h) > ih * wh + for it in range(nt) + if min((it + 1) * wt, t) > it * wt + ] + +def make_shifted_720Pwindows_bysize(size: Tuple[int, int, int], num_windows: Tuple[int, int, int]): + t, h, w = size + resized_nt, resized_nh, resized_nw = num_windows + #cal windows under 720p + scale = math.sqrt((45 * 80) / (h * w)) + resized_h, resized_w = round(h * scale), round(w * scale) + wh, ww = ceil(resized_h / resized_nh), ceil(resized_w / resized_nw) # window size. + wt = ceil(min(t, 30) / resized_nt) # window size. + + st, sh, sw = ( # shift size. + 0.5 if wt < t else 0, + 0.5 if wh < h else 0, + 0.5 if ww < w else 0, + ) + nt, nh, nw = ceil((t - st) / wt), ceil((h - sh) / wh), ceil((w - sw) / ww) # window size. + nt, nh, nw = ( # number of window. + nt + 1 if st > 0 else 1, + nh + 1 if sh > 0 else 1, + nw + 1 if sw > 0 else 1, + ) + return [ + ( + slice(max(int((it - st) * wt), 0), min(int((it - st + 1) * wt), t)), + slice(max(int((ih - sh) * wh), 0), min(int((ih - sh + 1) * wh), h)), + slice(max(int((iw - sw) * ww), 0), min(int((iw - sw + 1) * ww), w)), + ) + for iw in range(nw) + if min(int((iw - sw + 1) * ww), w) > max(int((iw - sw) * ww), 0) + for ih in range(nh) + if min(int((ih - sh + 1) * wh), h) > max(int((ih - sh) * wh), 0) + for it in range(nt) + if min(int((it - st + 1) * wt), t) > max(int((it - st) * wt), 0) + ] diff --git a/modules/seedvr/src/models/dit_v2/attention.py b/modules/seedvr/src/models/dit_v2/attention.py new file mode 100644 index 000000000..a88fd7d13 --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/attention.py @@ -0,0 +1,92 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +import torch +import torch.nn.functional as F + +#from flash_attn import flash_attn_varlen_func + +from torch import nn + + +def pytorch_varlen_attention(q, k, v, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, dropout_p=0.0, softmax_scale=None, causal=False, deterministic=False): + """ + A PyTorch-based implementation of variable-length attention to replace flash_attn_varlen_func. + It processes each sequence in the batch individually. + """ + # Create an empty tensor to store the output. + output = torch.empty_like(q) + + # Iterate over each sequence in the batch. The batch size is the number of sequences. + for i in range(len(cu_seqlens_q) - 1): + # Determine the start and end indices for the current sequence. + start_q, end_q = cu_seqlens_q[i], cu_seqlens_q[i+1] + start_k, end_k = cu_seqlens_k[i], cu_seqlens_k[i+1] + + # Slice the q, k, and v tensors to get the data for the current sequence. + # The shape is (seq_len, heads, head_dim). + q_i = q[start_q:end_q] + k_i = k[start_k:end_k] + v_i = v[start_k:end_k] + + # Reshape for torch's scaled_dot_product_attention which expects (batch, heads, seq, dim). + # Here, we treat each sequence as a batch of 1. + q_i = q_i.permute(1, 0, 2).unsqueeze(0) # (1, heads, seq_len_q, head_dim) + k_i = k_i.permute(1, 0, 2).unsqueeze(0) # (1, heads, seq_len_k, head_dim) + v_i = v_i.permute(1, 0, 2).unsqueeze(0) # (1, heads, seq_len_k, head_dim) + + # Use PyTorch's built-in scaled dot-product attention. + output_i = F.scaled_dot_product_attention( + q_i, k_i, v_i, + dropout_p=dropout_p if not deterministic else 0.0, + is_causal=causal + ) + + # Reshape the output back to the original format (seq_len, heads, head_dim) + output_i = output_i.squeeze(0).permute(1, 0, 2) + + # Place the result for the current sequence into the main output tensor. + output[start_q:end_q] = output_i + + return output + +class TorchAttention(nn.Module): + def tflops(self, args, kwargs, output) -> float: + assert len(args) == 0 or len(args) > 2, "query, key should both provided by args / kwargs" + q = kwargs.get("query") or args[0] + k = kwargs.get("key") or args[1] + b, h, sq, d = q.shape + b, h, sk, d = k.shape + return b * h * (4 * d * (sq / 1e6) * (sk / 1e6)) + + def forward(self, *args, **kwargs): + return F.scaled_dot_product_attention(*args, **kwargs) + + +class FlashAttentionVarlen(nn.Module): + def tflops(self, args, kwargs, output) -> float: + cu_seqlens_q = kwargs["cu_seqlens_q"] + cu_seqlens_k = kwargs["cu_seqlens_k"] + _, h, d = output.shape + seqlens_q = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]) / 1e6 + seqlens_k = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]) / 1e6 + return h * (4 * d * (seqlens_q * seqlens_k).sum()) + + def forward(self, *args, **kwargs): + kwargs["deterministic"] = torch.are_deterministic_algorithms_enabled() + try: + from flash_attn import flash_attn_varlen_func + return flash_attn_varlen_func(*args, **kwargs) + except ImportError: + return pytorch_varlen_attention(*args, **kwargs) diff --git a/modules/seedvr/src/models/dit_v2/embedding.py b/modules/seedvr/src/models/dit_v2/embedding.py new file mode 100644 index 000000000..e972244f5 --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/embedding.py @@ -0,0 +1,62 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Optional, Union +import torch +from diffusers.models.embeddings import get_timestep_embedding +from torch import nn + + +def emb_add(emb1: torch.Tensor, emb2: Optional[torch.Tensor]): + return emb1 if emb2 is None else emb1 + emb2 + + +class TimeEmbedding(nn.Module): + def __init__( + self, + sinusoidal_dim: int, + hidden_dim: int, + output_dim: int, + ): + super().__init__() + self.sinusoidal_dim = sinusoidal_dim + self.proj_in = nn.Linear(sinusoidal_dim, hidden_dim) + self.proj_hid = nn.Linear(hidden_dim, hidden_dim) + self.proj_out = nn.Linear(hidden_dim, output_dim) + self.act = nn.SiLU() + + def forward( + self, + timestep: Union[int, float, torch.IntTensor, torch.FloatTensor], + device: torch.device, + dtype: torch.dtype, + ) -> torch.FloatTensor: + if not torch.is_tensor(timestep): + timestep = torch.tensor([timestep], device=device, dtype=dtype) + if timestep.ndim == 0: + timestep = timestep[None] + + emb = get_timestep_embedding( + timesteps=timestep, + embedding_dim=self.sinusoidal_dim, + flip_sin_to_cos=False, + downscale_freq_shift=0, + ) + emb = emb.to(dtype) + emb = self.proj_in(emb) + emb = self.act(emb) + emb = self.proj_hid(emb) + emb = self.act(emb) + emb = self.proj_out(emb) + return emb diff --git a/modules/seedvr/src/models/dit_v2/mlp.py b/modules/seedvr/src/models/dit_v2/mlp.py new file mode 100644 index 000000000..2d05cb021 --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/mlp.py @@ -0,0 +1,62 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Optional +import torch +import torch.nn.functional as F +from torch import nn + + +def get_mlp(mlp_type: Optional[str] = "normal"): + if mlp_type == "normal": + return MLP + elif mlp_type == "swiglu": + return SwiGLUMLP + + +class MLP(nn.Module): + def __init__( + self, + dim: int, + expand_ratio: int, + ): + super().__init__() + self.proj_in = nn.Linear(dim, dim * expand_ratio) + self.act = nn.GELU("tanh") + self.proj_out = nn.Linear(dim * expand_ratio, dim) + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + x = self.proj_in(x) + x = self.act(x) + x = self.proj_out(x) + return x + + +class SwiGLUMLP(nn.Module): + def __init__( + self, + dim: int, + expand_ratio: int, + multiple_of: int = 256, + ): + super().__init__() + hidden_dim = int(2 * dim * expand_ratio / 3) + hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of) + self.proj_in_gate = nn.Linear(dim, hidden_dim, bias=False) + self.proj_out = nn.Linear(hidden_dim, dim, bias=False) + self.proj_in = nn.Linear(dim, hidden_dim, bias=False) + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + x = self.proj_out(F.silu(self.proj_in_gate(x)) * self.proj_in(x)) + return x diff --git a/modules/seedvr/src/models/dit_v2/mm.py b/modules/seedvr/src/models/dit_v2/mm.py new file mode 100644 index 000000000..344f89a8f --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/mm.py @@ -0,0 +1,74 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Tuple +import torch +from torch import nn + + +@dataclass +class MMArg: + vid: Any + txt: Any + + +def get_args(key: str, args: List[Any]) -> List[Any]: + return [getattr(v, key) if isinstance(v, MMArg) else v for v in args] + + +def get_kwargs(key: str, kwargs: Dict[str, Any]) -> Dict[str, Any]: + return {k: getattr(v, key) if isinstance(v, MMArg) else v for k, v in kwargs.items()} + + +class MMModule(nn.Module): + def __init__( + self, + module: Callable[..., nn.Module], + *args, + shared_weights: bool = False, + vid_only: bool = False, + **kwargs, + ): + super().__init__() + self.shared_weights = shared_weights + self.vid_only = vid_only + if self.shared_weights: + assert get_args("vid", args) == get_args("txt", args) + assert get_kwargs("vid", kwargs) == get_kwargs("txt", kwargs) + self.all = module(*get_args("vid", args), **get_kwargs("vid", kwargs)) + else: + self.vid = module(*get_args("vid", args), **get_kwargs("vid", kwargs)) + self.txt = ( + module(*get_args("txt", args), **get_kwargs("txt", kwargs)) + if not vid_only + else None + ) + + def forward( + self, + vid: torch.FloatTensor, + txt: torch.FloatTensor, + *args, + **kwargs, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + vid_module = self.vid if not self.shared_weights else self.all + vid = vid_module(vid, *get_args("vid", args), **get_kwargs("vid", kwargs)) + if not self.vid_only: + txt_module = self.txt if not self.shared_weights else self.all + txt = txt_module(txt, *get_args("txt", args), **get_kwargs("txt", kwargs)) + return vid, txt diff --git a/modules/seedvr/src/models/dit_v2/modulation.py b/modules/seedvr/src/models/dit_v2/modulation.py new file mode 100644 index 000000000..a10c49f3e --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/modulation.py @@ -0,0 +1,118 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Callable, List, Optional +import torch +from einops import rearrange +from torch import nn + +from ...common.cache import Cache +from ...common.distributed.ops import slice_inputs + +# (dim: int, emb_dim: int) +ada_layer_type = Callable[[int, int], nn.Module] + + +def get_ada_layer(ada_layer: str) -> ada_layer_type: + if ada_layer == "single": + return AdaSingle + raise NotImplementedError(f"{ada_layer} is not supported") + + +def expand_dims(x: torch.Tensor, dim: int, ndim: int): + """ + Expand tensor "x" to "ndim" by adding empty dims at "dim". + Example: x is (b d), target ndim is 5, add dim at 1, return (b 1 1 1 d). + """ + shape = x.shape + shape = shape[:dim] + (1,) * (ndim - len(shape)) + shape[dim:] + return x.reshape(shape) + + +class AdaSingle(nn.Module): + def __init__( + self, + dim: int, + emb_dim: int, + layers: List[str], + modes: List[str] = ["in", "out"], + ): + assert emb_dim == 6 * dim, "AdaSingle requires emb_dim == 6 * dim" + super().__init__() + self.dim = dim + self.emb_dim = emb_dim + self.layers = layers + for l in layers: + if "in" in modes: + self.register_parameter(f"{l}_shift", nn.Parameter(torch.randn(dim) / dim**0.5)) + self.register_parameter( + f"{l}_scale", nn.Parameter(torch.randn(dim) / dim**0.5 + 1) + ) + if "out" in modes: + self.register_parameter(f"{l}_gate", nn.Parameter(torch.randn(dim) / dim**0.5)) + + def forward( + self, + hid: torch.FloatTensor, # b ... c + emb: torch.FloatTensor, # b d + layer: str, + mode: str, + cache: Cache = Cache(disable=True), + branch_tag: str = "", + hid_len: Optional[torch.LongTensor] = None, # b + ) -> torch.FloatTensor: + idx = self.layers.index(layer) + emb = rearrange(emb, "b (d l g) -> b d l g", l=len(self.layers), g=3)[..., idx, :] + emb = expand_dims(emb, 1, hid.ndim + 1) + + if hid_len is not None: + emb = cache( + f"emb_repeat_{idx}_{branch_tag}", + lambda: slice_inputs( + torch.cat([e.repeat(l, *([1] * e.ndim)) for e, l in zip(emb, hid_len)]), + dim=0, + ), + ) + + shiftA, scaleA, gateA = emb.unbind(-1) + shiftB, scaleB, gateB = ( + getattr(self, f"{layer}_shift", None), + getattr(self, f"{layer}_scale", None), + getattr(self, f"{layer}_gate", None), + ) + + # 🚀 FP8 COMPATIBILITY: Convert parameters to match embedding dtype + # This prevents "Promotion for Float8 Types is not supported" errors + target_dtype = shiftA.dtype + + if mode == "in": + # Convert parameters to match embedding dtype for FP8 compatibility + if scaleB is not None and scaleB.dtype != target_dtype: + scaleB = scaleB.to(target_dtype) + if shiftB is not None and shiftB.dtype != target_dtype: + shiftB = shiftB.to(target_dtype) + + return hid.mul_(scaleA + scaleB).add_(shiftA + shiftB) + + if mode == "out": + # Convert gate parameter to match embedding dtype for FP8 compatibility + if gateB is not None and gateB.dtype != target_dtype: + gateB = gateB.to(target_dtype) + + return hid.mul_(gateA + gateB) + + raise NotImplementedError + + def extra_repr(self) -> str: + return f"dim={self.dim}, emb_dim={self.emb_dim}, layers={self.layers}" diff --git a/modules/seedvr/src/models/dit_v2/na.py b/modules/seedvr/src/models/dit_v2/na.py new file mode 100644 index 000000000..0dbd546c4 --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/na.py @@ -0,0 +1,241 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from itertools import chain +from typing import Callable, Dict, List, Tuple +import einops +import torch + + +def flatten( + hid: List[torch.FloatTensor], # List of (*** c) +) -> Tuple[ + torch.FloatTensor, # (L c) + torch.LongTensor, # (b n) +]: + assert len(hid) > 0 + shape = torch.stack([torch.tensor(x.shape[:-1], device=hid[0].device) for x in hid]) + hid = torch.cat([x.flatten(0, -2) for x in hid]) + return hid, shape + + +def unflatten( + hid: torch.FloatTensor, # (L c) or (L ... c) + hid_shape: torch.LongTensor, # (b n) +) -> List[torch.Tensor]: # List of (*** c) or (*** ... c) + hid_len = hid_shape.prod(-1) + hid = hid.split(hid_len.tolist()) + hid = [x.unflatten(0, s.tolist()) for x, s in zip(hid, hid_shape)] + return hid + + +def concat( + vid: torch.FloatTensor, # (VL ... c) + txt: torch.FloatTensor, # (TL ... c) + vid_len: torch.LongTensor, # (b) + txt_len: torch.LongTensor, # (b) +) -> torch.FloatTensor: # (L ... c) + vid = torch.split(vid, vid_len.tolist()) + txt = torch.split(txt, txt_len.tolist()) + return torch.cat(list(chain(*zip(vid, txt)))) + + +def concat_idx( + vid_len: torch.LongTensor, # (b) + txt_len: torch.LongTensor, # (b) +) -> Tuple[ + Callable, + Callable, +]: + device = vid_len.device + vid_idx = torch.arange(vid_len.sum(), device=device) + txt_idx = torch.arange(len(vid_idx), len(vid_idx) + txt_len.sum(), device=device) + tgt_idx = concat(vid_idx, txt_idx, vid_len, txt_len) + src_idx = torch.argsort(tgt_idx) + return ( + lambda vid, txt: torch.index_select(torch.cat([vid, txt]), 0, tgt_idx), + lambda all: torch.index_select(all, 0, src_idx).split([len(vid_idx), len(txt_idx)]), + ) + + +def unconcat( + all: torch.FloatTensor, # (L ... c) + vid_len: torch.LongTensor, # (b) + txt_len: torch.LongTensor, # (b) +) -> Tuple[ + torch.FloatTensor, # (VL ... c) + torch.FloatTensor, # (TL ... c) +]: + interleave_len = list(chain(*zip(vid_len.tolist(), txt_len.tolist()))) + all = all.split(interleave_len) + vid = torch.cat(all[0::2]) + txt = torch.cat(all[1::2]) + return vid, txt + + +def repeat_concat( + vid: torch.FloatTensor, # (VL ... c) + txt: torch.FloatTensor, # (TL ... c) + vid_len: torch.LongTensor, # (n*b) + txt_len: torch.LongTensor, # (b) + txt_repeat: List, # (n) +) -> torch.FloatTensor: # (L ... c) + vid = torch.split(vid, vid_len.tolist()) + txt = torch.split(txt, txt_len.tolist()) + txt = [[x] * n for x, n in zip(txt, txt_repeat)] + txt = list(chain(*txt)) + return torch.cat(list(chain(*zip(vid, txt)))) + + +def repeat_concat_idx( + vid_len: torch.LongTensor, # (n*b) + txt_len: torch.LongTensor, # (b) + txt_repeat: torch.LongTensor, # (n) +) -> Tuple[ + Callable, + Callable, +]: + device = vid_len.device + vid_idx = torch.arange(vid_len.sum(), device=device) + txt_idx = torch.arange(len(vid_idx), len(vid_idx) + txt_len.sum(), device=device) + txt_repeat_list = txt_repeat.tolist() + tgt_idx = repeat_concat(vid_idx, txt_idx, vid_len, txt_len, txt_repeat) + src_idx = torch.argsort(tgt_idx) + txt_idx_len = len(tgt_idx) - len(vid_idx) + repeat_txt_len = (txt_len * txt_repeat).tolist() + + def unconcat_coalesce(all): + """ + Un-concat vid & txt, and coalesce the repeated txt. + e.g. vid [0 1 2 3 4 5 6 7 8] -> 3 splits -> [0 1 2] [3 4 5] [6 7 8] + txt [9 10] + repeat_concat ==> [0 1 2 9 10 3 4 5 9 10 6 7 8 9 10] + 1. argsort re-index ==> [0 1 2 3 4 5 6 7 8 9 9 9 10 10 10] + split ==> vid_out [0 1 2 3 4 5 6 7 8] txt_out [9 9 9 10 10 10] + 2. reshape & mean for each sample to coalesce the repeated txt. + """ + vid_out, txt_out = all[src_idx].split([len(vid_idx), txt_idx_len]) + txt_out_coalesced = [] + for txt, repeat_time in zip(txt_out.split(repeat_txt_len), txt_repeat_list): + txt = txt.reshape(-1, repeat_time, *txt.shape[1:]).mean(1) + txt_out_coalesced.append(txt) + return vid_out, torch.cat(txt_out_coalesced) + + # Note: Backward of torch.index_select is non-deterministic when existing repeated index, + # the difference may cumulative like torch.repeat_interleave, so we use vanilla index here. + return ( + lambda vid, txt: torch.cat([vid, txt])[tgt_idx], + lambda all: unconcat_coalesce(all), + ) + + +def rearrange( + hid: torch.FloatTensor, # (L c) + hid_shape: torch.LongTensor, # (b n) + pattern: str, + **kwargs: Dict[str, int], +) -> Tuple[ + torch.FloatTensor, + torch.LongTensor, +]: + return flatten([einops.rearrange(h, pattern, **kwargs) for h in unflatten(hid, hid_shape)]) + + +def rearrange_idx( + hid_shape: torch.LongTensor, # (b n) + pattern: str, + **kwargs: Dict[str, int], +) -> Tuple[Callable, Callable, torch.LongTensor]: + hid_idx = torch.arange(hid_shape.prod(-1).sum(), device=hid_shape.device).unsqueeze(-1) + tgt_idx, tgt_shape = rearrange(hid_idx, hid_shape, pattern, **kwargs) + tgt_idx = tgt_idx.squeeze(-1) + src_idx = torch.argsort(tgt_idx) + return ( + lambda hid: torch.index_select(hid, 0, tgt_idx), + lambda hid: torch.index_select(hid, 0, src_idx), + tgt_shape, + ) + + +def repeat( + hid: torch.FloatTensor, # (L c) + hid_shape: torch.LongTensor, # (b n) + pattern: str, + **kwargs: Dict[str, torch.LongTensor], # (b) +) -> Tuple[ + torch.FloatTensor, + torch.LongTensor, +]: + hid = unflatten(hid, hid_shape) + kwargs = [{k: v[i].item() for k, v in kwargs.items()} for i in range(len(hid))] + return flatten([einops.repeat(h, pattern, **a) for h, a in zip(hid, kwargs)]) + + +def pack( + samples: List[torch.Tensor], # List of (h w c). +) -> Tuple[ + List[torch.Tensor], # groups [(b1 h1 w1 c1), (b2 h2 w2 c2)] + List[List[int]], # reversal indices. +]: + batches = {} + indices = {} + for i, sample in enumerate(samples): + shape = sample.shape + batches[shape] = batches.get(shape, []) + indices[shape] = indices.get(shape, []) + batches[shape].append(sample) + indices[shape].append(i) + + batches = list(map(torch.stack, batches.values())) + indices = list(indices.values()) + return batches, indices + + +def unpack( + batches: List[torch.Tensor], + indices: List[List[int]], +) -> List[torch.Tensor]: + samples = [None] * (max(chain(*indices)) + 1) + for batch, index in zip(batches, indices): + for sample, i in zip(batch.unbind(), index): + samples[i] = sample + return samples + + +def window( + hid: torch.FloatTensor, # (L c) + hid_shape: torch.LongTensor, # (b n) + window_fn: Callable[[torch.Tensor], List[torch.Tensor]], +): + hid = unflatten(hid, hid_shape) + hid = list(map(window_fn, hid)) + hid_windows = torch.tensor(list(map(len, hid)), device=hid_shape.device) + hid, hid_shape = flatten(list(chain(*hid))) + return hid, hid_shape, hid_windows + + +def window_idx( + hid_shape: torch.LongTensor, # (b n) + window_fn: Callable[[torch.Tensor], List[torch.Tensor]], +): + hid_idx = torch.arange(hid_shape.prod(-1).sum(), device=hid_shape.device).unsqueeze(-1) + tgt_idx, tgt_shape, tgt_windows = window(hid_idx, hid_shape, window_fn) + tgt_idx = tgt_idx.squeeze(-1) + src_idx = torch.argsort(tgt_idx) + return ( + lambda hid: torch.index_select(hid, 0, tgt_idx), + lambda hid: torch.index_select(hid, 0, src_idx), + tgt_shape, + tgt_windows, + ) diff --git a/modules/seedvr/src/models/dit_v2/nablocks/__init__.py b/modules/seedvr/src/models/dit_v2/nablocks/__init__.py new file mode 100644 index 000000000..c1a9da26e --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/nablocks/__init__.py @@ -0,0 +1,26 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from .mmsr_block import NaMMSRTransformerBlock + + +nadit_blocks = { + "mmdit_sr": NaMMSRTransformerBlock, +} + + +def get_nablock(block_type: str): + if block_type in nadit_blocks: + return nadit_blocks[block_type] + raise NotImplementedError(f"{block_type} is not supported") diff --git a/modules/seedvr/src/models/dit_v2/nablocks/attention/__init__.py b/modules/seedvr/src/models/dit_v2/nablocks/attention/__init__.py new file mode 100644 index 000000000..a75610252 --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/nablocks/attention/__init__.py @@ -0,0 +1,25 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from .mmattn import NaMMAttention + +attns = { + "mm_full": NaMMAttention, +} + + +def get_attn(attn_type: str): + if attn_type in attns: + return attns[attn_type] + raise NotImplementedError(f"{attn_type} is not supported") diff --git a/modules/seedvr/src/models/dit_v2/nablocks/attention/mmattn.py b/modules/seedvr/src/models/dit_v2/nablocks/attention/mmattn.py new file mode 100644 index 000000000..70f3bb90d --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/nablocks/attention/mmattn.py @@ -0,0 +1,267 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Optional, Tuple, Union +import torch +from einops import rearrange +from torch import nn +from torch.nn import functional as F +from torch.nn.modules.utils import _triple + +from .....common.cache import Cache +from .....common.distributed.ops import gather_heads_scatter_seq, gather_seq_scatter_heads_qkv +from .....common.half_precision_fixes import safe_pad_operation + +from ... import na +from ...attention import FlashAttentionVarlen +from ...mm import MMArg, MMModule +from ...normalization import norm_layer_type +from ...rope import get_na_rope +from ...window import get_window_op +from itertools import chain + + +class NaMMAttention(nn.Module): + def __init__( + self, + vid_dim: int, + txt_dim: int, + heads: int, + head_dim: int, + qk_bias: bool, + qk_norm: norm_layer_type, + qk_norm_eps: float, + rope_type: Optional[str], + rope_dim: int, + shared_weights: bool, + **kwargs, + ): + super().__init__() + dim = MMArg(vid_dim, txt_dim) + inner_dim = heads * head_dim + qkv_dim = inner_dim * 3 + self.head_dim = head_dim + self.proj_qkv = MMModule( + nn.Linear, dim, qkv_dim, bias=qk_bias, shared_weights=shared_weights + ) + self.proj_out = MMModule(nn.Linear, inner_dim, dim, shared_weights=shared_weights) + self.norm_q = MMModule( + qk_norm, + dim=head_dim, + eps=qk_norm_eps, + elementwise_affine=True, + shared_weights=shared_weights, + ) + self.norm_k = MMModule( + qk_norm, + dim=head_dim, + eps=qk_norm_eps, + elementwise_affine=True, + shared_weights=shared_weights, + ) + + self.rope = get_na_rope(rope_type=rope_type, dim=rope_dim) + self.attn = FlashAttentionVarlen() + + def forward( + self, + vid: torch.FloatTensor, # l c + txt: torch.FloatTensor, # l c + vid_shape: torch.LongTensor, # b 3 + txt_shape: torch.LongTensor, # b 1 + cache: Cache, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + vid_qkv, txt_qkv = self.proj_qkv(vid, txt) + vid_qkv = gather_seq_scatter_heads_qkv( + vid_qkv, + seq_dim=0, + qkv_shape=vid_shape, + cache=cache.namespace("vid"), + ) + txt_qkv = gather_seq_scatter_heads_qkv( + txt_qkv, + seq_dim=0, + qkv_shape=txt_shape, + cache=cache.namespace("txt"), + ) + vid_qkv = rearrange(vid_qkv, "l (o h d) -> l o h d", o=3, d=self.head_dim) + txt_qkv = rearrange(txt_qkv, "l (o h d) -> l o h d", o=3, d=self.head_dim) + + vid_q, vid_k, vid_v = vid_qkv.unbind(1) + txt_q, txt_k, txt_v = txt_qkv.unbind(1) + + vid_q, txt_q = self.norm_q(vid_q, txt_q) + vid_k, txt_k = self.norm_k(vid_k, txt_k) + + if self.rope: + if self.rope.mm: + vid_q, vid_k, txt_q, txt_k = self.rope( + vid_q, vid_k, vid_shape, txt_q, txt_k, txt_shape, cache + ) + else: + vid_q, vid_k = self.rope(vid_q, vid_k, vid_shape, cache) + + vid_len = cache("vid_len", lambda: vid_shape.prod(-1)) + txt_len = cache("txt_len", lambda: txt_shape.prod(-1)) + all_len = cache("all_len", lambda: vid_len + txt_len) + + concat, unconcat = cache("mm_pnp", lambda: na.concat_idx(vid_len, txt_len)) + + attn = self.attn( + q=concat(vid_q, txt_q).bfloat16(), + k=concat(vid_k, txt_k).bfloat16(), + v=concat(vid_v, txt_v).bfloat16(), + cu_seqlens_q=cache("mm_seqlens", lambda: safe_pad_operation(all_len.cumsum(0), (1, 0)).int()), + cu_seqlens_k=cache("mm_seqlens", lambda: safe_pad_operation(all_len.cumsum(0), (1, 0)).int()), + max_seqlen_q=cache("mm_maxlen", lambda: all_len.max().item()), + max_seqlen_k=cache("mm_maxlen", lambda: all_len.max().item()), + ).type_as(vid_q) + + attn = rearrange(attn, "l h d -> l (h d)") + vid_out, txt_out = unconcat(attn) + vid_out = gather_heads_scatter_seq(vid_out, head_dim=1, seq_dim=0) + txt_out = gather_heads_scatter_seq(txt_out, head_dim=1, seq_dim=0) + + vid_out, txt_out = self.proj_out(vid_out, txt_out) + return vid_out, txt_out + + +class NaSwinAttention(NaMMAttention): + def __init__( + self, + *args, + window: Union[int, Tuple[int, int, int]], + window_method: str, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.window = _triple(window) + self.window_method = window_method + assert all(map(lambda v: isinstance(v, int) and v >= 0, self.window)) + + self.window_op = get_window_op(window_method) + + def forward( + self, + vid: torch.FloatTensor, # l c + txt: torch.FloatTensor, # l c + vid_shape: torch.LongTensor, # b 3 + txt_shape: torch.LongTensor, # b 1 + cache: Cache, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + + vid_qkv, txt_qkv = self.proj_qkv(vid, txt) + vid_qkv = gather_seq_scatter_heads_qkv( + vid_qkv, + seq_dim=0, + qkv_shape=vid_shape, + cache=cache.namespace("vid"), + ) + txt_qkv = gather_seq_scatter_heads_qkv( + txt_qkv, + seq_dim=0, + qkv_shape=txt_shape, + cache=cache.namespace("txt"), + ) + + # re-org the input seq for window attn + cache_win = cache.namespace(f"{self.window_method}_{self.window}_sd3") + + def make_window(x: torch.Tensor): + t, h, w, _ = x.shape + window_slices = self.window_op((t, h, w), self.window) + return [x[st, sh, sw] for (st, sh, sw) in window_slices] + + window_partition, window_reverse, window_shape, window_count = cache_win( + "win_transform", + lambda: na.window_idx(vid_shape, make_window), + ) + vid_qkv_win = window_partition(vid_qkv) + + vid_qkv_win = rearrange(vid_qkv_win, "l (o h d) -> l o h d", o=3, d=self.head_dim) + txt_qkv = rearrange(txt_qkv, "l (o h d) -> l o h d", o=3, d=self.head_dim) + + vid_q, vid_k, vid_v = vid_qkv_win.unbind(1) + txt_q, txt_k, txt_v = txt_qkv.unbind(1) + + vid_q, txt_q = self.norm_q(vid_q, txt_q) + vid_k, txt_k = self.norm_k(vid_k, txt_k) + + txt_len = cache("txt_len", lambda: txt_shape.prod(-1)) + + vid_len_win = cache_win("vid_len", lambda: window_shape.prod(-1)) + txt_len_win = cache_win("txt_len", lambda: txt_len.repeat_interleave(window_count)) + all_len_win = cache_win("all_len", lambda: vid_len_win + txt_len_win) + concat_win, unconcat_win = cache_win( + "mm_pnp", lambda: na.repeat_concat_idx(vid_len_win, txt_len, window_count) + ) + + # window rope + if self.rope: + if self.rope.mm: + # repeat text q and k for window mmrope + _, num_h, _ = txt_q.shape + txt_q_repeat = rearrange(txt_q, "l h d -> l (h d)") + txt_q_repeat = na.unflatten(txt_q_repeat, txt_shape) + txt_q_repeat = [[x] * n for x, n in zip(txt_q_repeat, window_count)] + txt_q_repeat = list(chain(*txt_q_repeat)) + txt_q_repeat, txt_shape_repeat = na.flatten(txt_q_repeat) + txt_q_repeat = rearrange(txt_q_repeat, "l (h d) -> l h d", h=num_h) + + txt_k_repeat = rearrange(txt_k, "l h d -> l (h d)") + txt_k_repeat = na.unflatten(txt_k_repeat, txt_shape) + txt_k_repeat = [[x] * n for x, n in zip(txt_k_repeat, window_count)] + txt_k_repeat = list(chain(*txt_k_repeat)) + txt_k_repeat, _ = na.flatten(txt_k_repeat) + txt_k_repeat = rearrange(txt_k_repeat, "l (h d) -> l h d", h=num_h) + + vid_q, vid_k, txt_q, txt_k = self.rope( + vid_q, vid_k, window_shape, txt_q_repeat, txt_k_repeat, txt_shape_repeat, cache_win + ) + else: + vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win) + + out = self.attn( + q=concat_win(vid_q, txt_q).bfloat16(), + k=concat_win(vid_k, txt_k).bfloat16(), + v=concat_win(vid_v, txt_v).bfloat16(), + cu_seqlens_q=cache_win( + "vid_seqlens_q", lambda: safe_pad_operation(all_len_win.cumsum(0), (1, 0)).int() + ), + cu_seqlens_k=cache_win( + "vid_seqlens_k", lambda: safe_pad_operation(all_len_win.cumsum(0), (1, 0)).int() + ), + max_seqlen_q=cache_win("vid_max_seqlen_q", lambda: all_len_win.max().item()), + max_seqlen_k=cache_win("vid_max_seqlen_k", lambda: all_len_win.max().item()), + ).type_as(vid_q) + + # text pooling + vid_out, txt_out = unconcat_win(out) + + vid_out = rearrange(vid_out, "l h d -> l (h d)") + txt_out = rearrange(txt_out, "l h d -> l (h d)") + vid_out = window_reverse(vid_out) + + vid_out = gather_heads_scatter_seq(vid_out, head_dim=1, seq_dim=0) + txt_out = gather_heads_scatter_seq(txt_out, head_dim=1, seq_dim=0) + + vid_out, txt_out = self.proj_out(vid_out, txt_out) + + return vid_out, txt_out diff --git a/modules/seedvr/src/models/dit_v2/nablocks/mmsr_block.py b/modules/seedvr/src/models/dit_v2/nablocks/mmsr_block.py new file mode 100644 index 000000000..52cb67e1f --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/nablocks/mmsr_block.py @@ -0,0 +1,126 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Tuple +import torch +import torch.nn as nn + +# from ..cache import Cache +from ....common.cache import Cache + +from .attention.mmattn import NaSwinAttention +from ..mm import MMArg +from ..modulation import ada_layer_type +from ..normalization import norm_layer_type +from ..mm import MMModule +from ..mlp import get_mlp + + +class NaMMSRTransformerBlock(nn.Module): + def __init__( + self, + *, + vid_dim: int, + txt_dim: int, + emb_dim: int, + heads: int, + head_dim: int, + expand_ratio: int, + norm: norm_layer_type, + norm_eps: float, + ada: ada_layer_type, + qk_bias: bool, + qk_norm: norm_layer_type, + mlp_type: str, + shared_weights: bool, + rope_type: str, + rope_dim: int, + is_last_layer: bool, + **kwargs, + ): + super().__init__() + dim = MMArg(vid_dim, txt_dim) + self.attn_norm = MMModule(norm, dim=dim, eps=norm_eps, elementwise_affine=False, shared_weights=shared_weights,) + + self.attn = NaSwinAttention( + vid_dim=vid_dim, + txt_dim=txt_dim, + heads=heads, + head_dim=head_dim, + qk_bias=qk_bias, + qk_norm=qk_norm, + qk_norm_eps=norm_eps, + rope_type=rope_type, + rope_dim=rope_dim, + shared_weights=shared_weights, + window=kwargs.pop("window", None), + window_method=kwargs.pop("window_method", None), + ) + + self.mlp_norm = MMModule(norm, dim=dim, eps=norm_eps, elementwise_affine=False, shared_weights=shared_weights, vid_only=is_last_layer) + self.mlp = MMModule( + get_mlp(mlp_type), + dim=dim, + expand_ratio=expand_ratio, + shared_weights=shared_weights, + vid_only=is_last_layer + ) + self.ada = MMModule(ada, dim=dim, emb_dim=emb_dim, layers=["attn", "mlp"], shared_weights=shared_weights, vid_only=is_last_layer) + self.is_last_layer = is_last_layer + + def forward( + self, + vid: torch.FloatTensor, # l c + txt: torch.FloatTensor, # l c + vid_shape: torch.LongTensor, # b 3 + txt_shape: torch.LongTensor, # b 1 + emb: torch.FloatTensor, + cache: Cache, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + torch.LongTensor, + torch.LongTensor, + ]: + hid_len = MMArg( + cache("vid_len", lambda: vid_shape.prod(-1)), + cache("txt_len", lambda: txt_shape.prod(-1)), + ) + ada_kwargs = { + "emb": emb, + "hid_len": hid_len, + "cache": cache, + "branch_tag": MMArg("vid", "txt"), + } + + vid_attn, txt_attn = self.attn_norm(vid, txt) + + vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="in", **ada_kwargs) + vid_attn, txt_attn = self.attn(vid_attn, txt_attn, vid_shape, txt_shape, cache) + vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="out", **ada_kwargs) + vid_attn, txt_attn = (vid_attn + vid), (txt_attn + txt) + + vid_mlp, txt_mlp = self.mlp_norm(vid_attn, txt_attn) + # ADD BY NUMZ + if vid_mlp.dtype != vid_attn.dtype: + vid_mlp = vid_mlp.to(vid_attn.dtype) + if txt_mlp.dtype != txt_attn.dtype: + txt_mlp = txt_mlp.to(txt_attn.dtype) + # END BY NUMZ + vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="in", **ada_kwargs) + vid_mlp, txt_mlp = self.mlp(vid_mlp, txt_mlp) + vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="out", **ada_kwargs) + vid_mlp, txt_mlp = (vid_mlp + vid_attn), (txt_mlp + txt_attn) + + return vid_mlp, txt_mlp, vid_shape, txt_shape diff --git a/modules/seedvr/src/models/dit_v2/nadit.py b/modules/seedvr/src/models/dit_v2/nadit.py new file mode 100644 index 000000000..1f37ef8c3 --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/nadit.py @@ -0,0 +1,246 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union, Callable +import torch +from torch import nn + +from ...common.cache import Cache +from ...common.distributed.ops import slice_inputs + +from . import na +from .embedding import TimeEmbedding +from .modulation import get_ada_layer +from .nablocks import get_nablock +from .normalization import get_norm_layer +from .patch import get_na_patch_layers + +# Fake func, no checkpointing is required for inference +def gradient_checkpointing(module: Union[Callable, nn.Module], *args, enabled: bool, **kwargs): + return module(*args, **kwargs) + +@dataclass +class NaDiTOutput: + vid_sample: torch.Tensor + + +class NaDiT(nn.Module): + """ + Native Resolution Diffusion Transformer (NaDiT) + """ + + gradient_checkpointing = False + + def __init__( + self, + vid_in_channels: int, + vid_out_channels: int, + vid_dim: int, + txt_in_dim: Union[int, List[int]], + txt_dim: Optional[int], + emb_dim: int, + heads: int, + head_dim: int, + expand_ratio: int, + norm: Optional[str], + norm_eps: float, + ada: str, + qk_bias: bool, + qk_norm: Optional[str], + patch_size: Union[int, Tuple[int, int, int]], + num_layers: int, + block_type: Union[str, Tuple[str]], + mm_layers: Union[int, Tuple[bool]], + mlp_type: str = "normal", + patch_type: str = "v1", + rope_type: Optional[str] = "rope3d", + rope_dim: Optional[int] = None, + window: Optional[Tuple] = None, + window_method: Optional[Tuple[str]] = None, + msa_type: Optional[Tuple[str]] = None, + mca_type: Optional[Tuple[str]] = None, + txt_in_norm: Optional[str] = None, + txt_in_norm_scale_factor: int = 0.01, + txt_proj_type: Optional[str] = "linear", + vid_out_norm: Optional[str] = None, + **kwargs, + ): + ada = get_ada_layer(ada) + norm = get_norm_layer(norm) + qk_norm = get_norm_layer(qk_norm) + rope_dim = rope_dim if rope_dim is not None else head_dim // 2 + if isinstance(block_type, str): + block_type = [block_type] * num_layers + elif len(block_type) != num_layers: + raise ValueError("The ``block_type`` list should equal to ``num_layers``.") + super().__init__() + NaPatchIn, NaPatchOut = get_na_patch_layers(patch_type) + self.vid_in = NaPatchIn( + in_channels=vid_in_channels, + patch_size=patch_size, + dim=vid_dim, + ) + if not isinstance(txt_in_dim, int): + self.txt_in = nn.ModuleList([]) + for in_dim in txt_in_dim: + txt_norm_layer = get_norm_layer(txt_in_norm)(txt_dim, norm_eps, True) + if txt_proj_type == "linear": + txt_proj_layer = nn.Linear(in_dim, txt_dim) + else: + txt_proj_layer = nn.Sequential( + nn.Linear(in_dim, in_dim), nn.GELU("tanh"), nn.Linear(in_dim, txt_dim) + ) + torch.nn.init.constant_(txt_norm_layer.weight, txt_in_norm_scale_factor) + self.txt_in.append( + nn.Sequential( + txt_proj_layer, + txt_norm_layer, + ) + ) + else: + self.txt_in = ( + nn.Linear(txt_in_dim, txt_dim) + if txt_in_dim and txt_in_dim != txt_dim + else nn.Identity() + ) + self.emb_in = TimeEmbedding( + sinusoidal_dim=256, + hidden_dim=max(vid_dim, txt_dim), + output_dim=emb_dim, + ) + + if window is None or isinstance(window[0], int): + window = [window] * num_layers + if window_method is None or isinstance(window_method, str): + window_method = [window_method] * num_layers + + if msa_type is None or isinstance(msa_type, str): + msa_type = [msa_type] * num_layers + if mca_type is None or isinstance(mca_type, str): + mca_type = [mca_type] * num_layers + + self.blocks = nn.ModuleList( + [ + get_nablock(block_type[i])( + vid_dim=vid_dim, + txt_dim=txt_dim, + emb_dim=emb_dim, + heads=heads, + head_dim=head_dim, + expand_ratio=expand_ratio, + norm=norm, + norm_eps=norm_eps, + ada=ada, + qk_bias=qk_bias, + qk_norm=qk_norm, + shared_weights=not ( + (i < mm_layers) if isinstance(mm_layers, int) else mm_layers[i] + ), + mlp_type=mlp_type, + window=window[i], + window_method=window_method[i], + msa_type=msa_type[i], + mca_type=mca_type[i], + rope_type=rope_type, + rope_dim=rope_dim, + is_last_layer=(i == num_layers - 1), + **kwargs, + ) + for i in range(num_layers) + ] + ) + + self.vid_out_norm = None + if vid_out_norm is not None: + self.vid_out_norm = get_norm_layer(vid_out_norm)( + dim=vid_dim, + eps=norm_eps, + elementwise_affine=True, + ) + self.vid_out_ada = ada( + dim=vid_dim, + emb_dim=emb_dim, + layers=["out"], + modes=["in"], + ) + + self.vid_out = NaPatchOut( + out_channels=vid_out_channels, + patch_size=patch_size, + dim=vid_dim, + ) + + def set_gradient_checkpointing(self, enable: bool): + self.gradient_checkpointing = enable + + def forward( + self, + vid: torch.FloatTensor, # l c + txt: Union[torch.FloatTensor, List[torch.FloatTensor]], # l c + vid_shape: torch.LongTensor, # b 3 + txt_shape: Union[torch.LongTensor, List[torch.LongTensor]], # b 1 + timestep: Union[int, float, torch.IntTensor, torch.FloatTensor], # b + disable_cache: bool = False, # for test + ): + cache = Cache(disable=disable_cache) + + # slice vid after patching in when using sequence parallelism + if isinstance(txt, list): + assert isinstance(self.txt_in, nn.ModuleList) + txt = [ + na.unflatten(fc(i), s) for fc, i, s in zip(self.txt_in, txt, txt_shape) + ] # B L D + txt, txt_shape = na.flatten([torch.cat(t, dim=0) for t in zip(*txt)]) + txt = slice_inputs(txt, dim=0) + else: + txt = slice_inputs(txt, dim=0) + txt = self.txt_in(txt) + + # Video input. + # Sequence parallel slicing is done inside patching class. + vid, vid_shape = self.vid_in(vid, vid_shape, cache) + + # Embedding input. + emb = self.emb_in(timestep, device=vid.device, dtype=vid.dtype) + + # Body + for i, block in enumerate(self.blocks): + vid, txt, vid_shape, txt_shape = gradient_checkpointing( + enabled=(self.gradient_checkpointing and self.training), + module=block, + vid=vid, + txt=txt, + vid_shape=vid_shape, + txt_shape=txt_shape, + emb=emb, + cache=cache, + ) + + # Video output norm. + if self.vid_out_norm: + vid = self.vid_out_norm(vid) + vid = self.vid_out_ada( + vid, + emb=emb, + layer="out", + mode="in", + hid_len=cache("vid_len", lambda: vid_shape.prod(-1)), + cache=cache, + branch_tag="vid", + ) + + # Video output. + vid, vid_shape = self.vid_out(vid, vid_shape, cache) + return NaDiTOutput(vid_sample=vid) diff --git a/modules/seedvr/src/models/dit_v2/normalization.py b/modules/seedvr/src/models/dit_v2/normalization.py new file mode 100644 index 000000000..41ba35a39 --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/normalization.py @@ -0,0 +1,147 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Callable, Optional +from diffusers.models.normalization import RMSNorm +from torch import nn +import torch +import torch.nn.functional as F +import numbers +from torch.nn.parameter import Parameter +from torch.nn import init + +# (dim: int, eps: float, elementwise_affine: bool) +norm_layer_type = Callable[[int, float, bool], nn.Module] + + +class CustomLayerNorm(nn.Module): + """ + Custom LayerNorm implementation to replace Apex FusedLayerNorm + """ + def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True): + super(CustomLayerNorm, self).__init__() + + if isinstance(normalized_shape, numbers.Integral): + normalized_shape = (normalized_shape,) + self.normalized_shape = torch.Size(normalized_shape) + self.eps = eps + self.elementwise_affine = elementwise_affine + + if self.elementwise_affine: + self.weight = Parameter(torch.Tensor(*normalized_shape)) + self.bias = Parameter(torch.Tensor(*normalized_shape)) + else: + self.register_parameter('weight', None) + self.register_parameter('bias', None) + self.reset_parameters() + + def reset_parameters(self): + if self.elementwise_affine: + init.ones_(self.weight) + init.zeros_(self.bias) + + def forward(self, input): + # 🚀 FP8 COMPATIBILITY: Convert parameters to match input dtype + # This prevents "Promotion for Float8 Types is not supported" errors + weight = self.weight + bias = self.bias + + if self.elementwise_affine and weight is not None: + if weight.dtype != input.dtype: + weight = weight.to(input.dtype) + if bias is not None and bias.dtype != input.dtype: + bias = bias.to(input.dtype) + + return F.layer_norm( + input, self.normalized_shape, weight, bias, self.eps) + + +class CustomRMSNorm(nn.Module): + """ + Custom RMSNorm implementation to replace Apex FusedRMSNorm + """ + def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True): + super(CustomRMSNorm, self).__init__() + + if isinstance(normalized_shape, numbers.Integral): + normalized_shape = (normalized_shape,) + self.normalized_shape = torch.Size(normalized_shape) + self.eps = eps + self.elementwise_affine = elementwise_affine + + if self.elementwise_affine: + self.weight = Parameter(torch.ones(*normalized_shape)) + else: + self.register_parameter('weight', None) + + def forward(self, input): + # RMS normalization: x / sqrt(mean(x^2) + eps) * weight + dims = tuple(range(-len(self.normalized_shape), 0)) + + # Calculate RMS: sqrt(mean(x^2)) + variance = input.pow(2).mean(dim=dims, keepdim=True) + rms = torch.sqrt(variance + self.eps) + + # Normalize + normalized = input / rms + + if self.elementwise_affine: + # 🚀 FP8 COMPATIBILITY: Convert weight to match normalized dtype + # This prevents "Promotion for Float8 Types is not supported" errors + weight = self.weight + if weight.dtype != normalized.dtype: + weight = weight.to(normalized.dtype) + return normalized * weight + return normalized + + +def get_norm_layer(norm_type: Optional[str]) -> norm_layer_type: + + def _norm_layer(dim: int, eps: float, elementwise_affine: bool): + if norm_type is None: + return nn.Identity() + + if norm_type == "layer": + return nn.LayerNorm( + normalized_shape=dim, + eps=eps, + elementwise_affine=elementwise_affine, + ) + + if norm_type == "rms": + return RMSNorm( + dim=dim, + eps=eps, + elementwise_affine=elementwise_affine, + ) + + if norm_type == "fusedln": + # Use custom LayerNorm instead of Apex FusedLayerNorm + return CustomLayerNorm( + normalized_shape=dim, + elementwise_affine=elementwise_affine, + eps=eps, + ) + + if norm_type == "fusedrms": + # Use custom RMSNorm instead of Apex FusedRMSNorm + return CustomRMSNorm( + normalized_shape=dim, + elementwise_affine=elementwise_affine, + eps=eps, + ) + + raise NotImplementedError(f"{norm_type} is not supported") + + return _norm_layer diff --git a/modules/seedvr/src/models/dit_v2/patch/__init__.py b/modules/seedvr/src/models/dit_v2/patch/__init__.py new file mode 100644 index 000000000..4e3c97831 --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/patch/__init__.py @@ -0,0 +1,19 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +def get_na_patch_layers(patch_type="v1"): + assert patch_type in ["v1"] + if patch_type == "v1": + from .patch_v1 import NaPatchIn, NaPatchOut + return NaPatchIn, NaPatchOut diff --git a/modules/seedvr/src/models/dit_v2/patch/patch_v1.py b/modules/seedvr/src/models/dit_v2/patch/patch_v1.py new file mode 100644 index 000000000..14b5a62d5 --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/patch/patch_v1.py @@ -0,0 +1,127 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Tuple, Union +import torch +from einops import rearrange +from torch import nn +from torch.nn.modules.utils import _triple + +from ....common.cache import Cache +from ....common.distributed.ops import gather_outputs, slice_inputs + +from .. import na + + +class PatchIn(nn.Module): + def __init__( + self, + in_channels: int, + patch_size: Union[int, Tuple[int, int, int]], + dim: int, + ): + super().__init__() + t, h, w = _triple(patch_size) + self.patch_size = t, h, w + self.proj = nn.Linear(in_channels * t * h * w, dim) + + def forward( + self, + vid: torch.Tensor, + ) -> torch.Tensor: + t, h, w = self.patch_size + if t > 1: + assert vid.size(2) % t == 1 + vid = torch.cat([vid[:, :, :1]] * (t - 1) + [vid], dim=2) + vid = rearrange(vid, "b c (T t) (H h) (W w) -> b T H W (t h w c)", t=t, h=h, w=w) + vid = self.proj(vid) + return vid + + +class PatchOut(nn.Module): + def __init__( + self, + out_channels: int, + patch_size: Union[int, Tuple[int, int, int]], + dim: int, + ): + super().__init__() + t, h, w = _triple(patch_size) + self.patch_size = t, h, w + self.proj = nn.Linear(dim, out_channels * t * h * w) + + def forward( + self, + vid: torch.Tensor, + ) -> torch.Tensor: + t, h, w = self.patch_size + vid = self.proj(vid) + vid = rearrange(vid, "b T H W (t h w c) -> b c (T t) (H h) (W w)", t=t, h=h, w=w) + if t > 1: + vid = vid[:, :, (t - 1) :] + return vid + + +class NaPatchIn(PatchIn): + def forward( + self, + vid: torch.Tensor, # l c + vid_shape: torch.LongTensor, + cache: Cache = Cache(disable=True), # for test + ) -> torch.Tensor: + cache = cache.namespace("patch") + vid_shape_before_patchify = cache("vid_shape_before_patchify", lambda: vid_shape) + t, h, w = self.patch_size + if not (t == h == w == 1): + vid = na.unflatten(vid, vid_shape) + for i in range(len(vid)): + if t > 1 and vid_shape_before_patchify[i, 0] % t != 0: + vid[i] = torch.cat([vid[i][:1]] * (t - vid[i].size(0) % t) + [vid[i]], dim=0) + vid[i] = rearrange(vid[i], "(T t) (H h) (W w) c -> T H W (t h w c)", t=t, h=h, w=w) + vid, vid_shape = na.flatten(vid) + + # slice vid after patching in when using sequence parallelism + vid = slice_inputs(vid, dim=0) + vid = self.proj(vid) + return vid, vid_shape + + +class NaPatchOut(PatchOut): + def forward( + self, + vid: torch.FloatTensor, # l c + vid_shape: torch.LongTensor, + cache: Cache = Cache(disable=True), # for test + ) -> Tuple[ + torch.FloatTensor, + torch.LongTensor, + ]: + cache = cache.namespace("patch") + vid_shape_before_patchify = cache.get("vid_shape_before_patchify") + + t, h, w = self.patch_size + vid = self.proj(vid) + # gather vid before patching out when enabling sequence parallelism + vid = gather_outputs( + vid, gather_dim=0, padding_dim=0, unpad_shape=vid_shape, cache=cache.namespace("vid") + ) + if not (t == h == w == 1): + vid = na.unflatten(vid, vid_shape) + for i in range(len(vid)): + vid[i] = rearrange(vid[i], "T H W (t h w c) -> (T t) (H h) (W w) c", t=t, h=h, w=w) + if t > 1 and vid_shape_before_patchify[i, 0] % t != 0: + vid[i] = vid[i][(t - vid_shape_before_patchify[i, 0] % t) :] + vid, vid_shape = na.flatten(vid) + + return vid, vid_shape diff --git a/modules/seedvr/src/models/dit_v2/rope.py b/modules/seedvr/src/models/dit_v2/rope.py new file mode 100644 index 000000000..3d294621c --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/rope.py @@ -0,0 +1,154 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from functools import lru_cache +from typing import Optional, Tuple +import torch +from einops import rearrange +from torch import nn +from ...common.cache import Cache +from ....rotary_embedding import RotaryEmbedding, apply_rotary_emb + + +class RotaryEmbeddingBase(nn.Module): + def __init__(self, dim: int, rope_dim: int): + super().__init__() + self.rope = RotaryEmbedding( + dim=dim // rope_dim, + freqs_for="pixel", + max_freq=256, + ) + # 1. Set model.requires_grad_(True) after model creation will make + # the `requires_grad=False` for rope freqs no longer hold. + # 2. Even if we don't set requires_grad_(True) explicitly, + # FSDP is not memory efficient when handling fsdp_wrap + # with mixed requires_grad=True/False. + # With above consideration, it is easier just remove the freqs + # out of nn.Parameters when `learned_freq=False` + freqs = self.rope.freqs + del self.rope.freqs + self.rope.register_buffer("freqs", freqs.data) + + @lru_cache(maxsize=128) + def get_axial_freqs(self, *dims): + return self.rope.get_axial_freqs(*dims) + + +class RotaryEmbedding3d(RotaryEmbeddingBase): + def __init__(self, dim: int): + super().__init__(dim, rope_dim=3) + self.mm = False + + def forward( + self, + q: torch.FloatTensor, # b h l d + k: torch.FloatTensor, # b h l d + size: Tuple[int, int, int], + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + ]: + T, H, W = size + freqs = self.get_axial_freqs(T, H, W) + q = rearrange(q, "b h (T H W) d -> b h T H W d", T=T, H=H, W=W) + k = rearrange(k, "b h (T H W) d -> b h T H W d", T=T, H=H, W=W) + q = apply_rotary_emb(freqs, q.float()).to(q.dtype) + k = apply_rotary_emb(freqs, k.float()).to(k.dtype) + q = rearrange(q, "b h T H W d -> b h (T H W) d") + k = rearrange(k, "b h T H W d -> b h (T H W) d") + return q, k + + +class MMRotaryEmbeddingBase(RotaryEmbeddingBase): + def __init__(self, dim: int, rope_dim: int): + super().__init__(dim, rope_dim) + self.rope = RotaryEmbedding( + dim=dim // rope_dim, + freqs_for="lang", + theta=10000, + ) + freqs = self.rope.freqs + del self.rope.freqs + self.rope.register_buffer("freqs", freqs.data) + self.mm = True + + +class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase): + def __init__(self, dim: int): + super().__init__(dim, rope_dim=3) + + def forward( + self, + vid_q: torch.FloatTensor, # L h d + vid_k: torch.FloatTensor, # L h d + vid_shape: torch.LongTensor, # B 3 + txt_q: torch.FloatTensor, # L h d + txt_k: torch.FloatTensor, # L h d + txt_shape: torch.LongTensor, # B 1 + cache: Cache, + ) -> Tuple[ + torch.FloatTensor, + torch.FloatTensor, + torch.FloatTensor, + torch.FloatTensor, + ]: + vid_freqs, txt_freqs = cache( + "mmrope_freqs_3d", + lambda: self.get_freqs(vid_shape, txt_shape), + ) + target_device = vid_q.device + if vid_freqs.device != target_device: + vid_freqs = vid_freqs.to(target_device) + if txt_freqs.device != target_device: + txt_freqs = txt_freqs.to(target_device) + vid_q = rearrange(vid_q, "L h d -> h L d") + vid_k = rearrange(vid_k, "L h d -> h L d") + vid_q = apply_rotary_emb(vid_freqs, vid_q.float()).to(vid_q.dtype) + vid_k = apply_rotary_emb(vid_freqs, vid_k.float()).to(vid_k.dtype) + vid_q = rearrange(vid_q, "h L d -> L h d") + vid_k = rearrange(vid_k, "h L d -> L h d") + + txt_q = rearrange(txt_q, "L h d -> h L d") + txt_k = rearrange(txt_k, "L h d -> h L d") + txt_q = apply_rotary_emb(txt_freqs, txt_q.float()).to(txt_q.dtype) + txt_k = apply_rotary_emb(txt_freqs, txt_k.float()).to(txt_k.dtype) + txt_q = rearrange(txt_q, "h L d -> L h d") + txt_k = rearrange(txt_k, "h L d -> L h d") + return vid_q, vid_k, txt_q, txt_k + + def get_freqs( + self, + vid_shape: torch.LongTensor, + txt_shape: torch.LongTensor, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + ]: + vid_freqs = self.get_axial_freqs(1024, 128, 128) + txt_freqs = self.get_axial_freqs(1024) + vid_freq_list, txt_freq_list = [], [] + for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()): + vid_freq = vid_freqs[l : l + f, :h, :w].reshape(-1, vid_freqs.size(-1)) + txt_freq = txt_freqs[:l].repeat(1, 3).reshape(-1, vid_freqs.size(-1)) + vid_freq_list.append(vid_freq) + txt_freq_list.append(txt_freq) + return torch.cat(vid_freq_list, dim=0), torch.cat(txt_freq_list, dim=0) + + +def get_na_rope(rope_type: Optional[str], dim: int): + if rope_type is None: + return None + if rope_type == "mmrope3d": + return NaMMRotaryEmbedding3d(dim=dim) + raise NotImplementedError(f"{rope_type} is not supported.") diff --git a/modules/seedvr/src/models/dit_v2/window.py b/modules/seedvr/src/models/dit_v2/window.py new file mode 100644 index 000000000..f77ea57d7 --- /dev/null +++ b/modules/seedvr/src/models/dit_v2/window.py @@ -0,0 +1,83 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from math import ceil +from typing import Tuple +import math + +def get_window_op(name: str): + if name == "720pwin_by_size_bysize": + return make_720Pwindows_bysize + if name == "720pswin_by_size_bysize": + return make_shifted_720Pwindows_bysize + raise ValueError(f"Unknown windowing method: {name}") + + +# -------------------------------- Windowing -------------------------------- # +def make_720Pwindows_bysize(size: Tuple[int, int, int], num_windows: Tuple[int, int, int]): + t, h, w = size + resized_nt, resized_nh, resized_nw = num_windows + #cal windows under 720p + scale = math.sqrt((45 * 80) / (h * w)) + resized_h, resized_w = round(h * scale), round(w * scale) + wh, ww = ceil(resized_h / resized_nh), ceil(resized_w / resized_nw) # window size. + wt = ceil(min(t, 30) / resized_nt) # window size. + nt, nh, nw = ceil(t / wt), ceil(h / wh), ceil(w / ww) # window size. + return [ + ( + slice(it * wt, min((it + 1) * wt, t)), + slice(ih * wh, min((ih + 1) * wh, h)), + slice(iw * ww, min((iw + 1) * ww, w)), + ) + for iw in range(nw) + if min((iw + 1) * ww, w) > iw * ww + for ih in range(nh) + if min((ih + 1) * wh, h) > ih * wh + for it in range(nt) + if min((it + 1) * wt, t) > it * wt + ] + +def make_shifted_720Pwindows_bysize(size: Tuple[int, int, int], num_windows: Tuple[int, int, int]): + t, h, w = size + resized_nt, resized_nh, resized_nw = num_windows + #cal windows under 720p + scale = math.sqrt((45 * 80) / (h * w)) + resized_h, resized_w = round(h * scale), round(w * scale) + wh, ww = ceil(resized_h / resized_nh), ceil(resized_w / resized_nw) # window size. + wt = ceil(min(t, 30) / resized_nt) # window size. + + st, sh, sw = ( # shift size. + 0.5 if wt < t else 0, + 0.5 if wh < h else 0, + 0.5 if ww < w else 0, + ) + nt, nh, nw = ceil((t - st) / wt), ceil((h - sh) / wh), ceil((w - sw) / ww) # window size. + nt, nh, nw = ( # number of window. + nt + 1 if st > 0 else 1, + nh + 1 if sh > 0 else 1, + nw + 1 if sw > 0 else 1, + ) + return [ + ( + slice(max(int((it - st) * wt), 0), min(int((it - st + 1) * wt), t)), + slice(max(int((ih - sh) * wh), 0), min(int((ih - sh + 1) * wh), h)), + slice(max(int((iw - sw) * ww), 0), min(int((iw - sw + 1) * ww), w)), + ) + for iw in range(nw) + if min(int((iw - sw + 1) * ww), w) > max(int((iw - sw) * ww), 0) + for ih in range(nh) + if min(int((ih - sh + 1) * wh), h) > max(int((ih - sh) * wh), 0) + for it in range(nt) + if min(int((it - st + 1) * wt), t) > max(int((it - st) * wt), 0) + ] diff --git a/modules/seedvr/src/models/video_vae_v3/modules/attn_video_vae.py b/modules/seedvr/src/models/video_vae_v3/modules/attn_video_vae.py new file mode 100644 index 000000000..2b07fb1d3 --- /dev/null +++ b/modules/seedvr/src/models/video_vae_v3/modules/attn_video_vae.py @@ -0,0 +1,1385 @@ +# Copyright (c) 2023 HuggingFace Team +# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates. +# SPDX-License-Identifier: Apache License, Version 2.0 (the "License") +# +# This file has been modified by ByteDance Ltd. and/or its affiliates. on 1st June 2025 +# +# Original file was released under Apache License, Version 2.0 (the "License"), with the full license text +# available at http://www.apache.org/licenses/LICENSE-2.0. +# +# This modified file is released under the same license. + + +from contextlib import nullcontext +from typing import Literal, Optional, Tuple, Union +import diffusers +import torch +import torch.nn as nn +from diffusers.models.attention_processor import Attention, SpatialNorm +from diffusers.models.autoencoders.vae import DecoderOutput, DiagonalGaussianDistribution +from diffusers.models.downsampling import Downsample2D +from diffusers.models.lora import LoRACompatibleConv +from diffusers.models.modeling_outputs import AutoencoderKLOutput +from diffusers.models.resnet import ResnetBlock2D +from diffusers.models.unets.unet_2d_blocks import DownEncoderBlock2D, UpDecoderBlock2D +from diffusers.models.upsampling import Upsample2D +from diffusers.utils import is_torch_version +from diffusers.utils.accelerate_utils import apply_forward_hook +from einops import rearrange +from ....common.half_precision_fixes import safe_pad_operation, safe_interpolate_operation +from ....common.logger import get_logger +from .causal_inflation_lib import InflatedCausalConv3d, causal_norm_wrapper, init_causal_conv3d, remove_head +from .context_parallel_lib import causal_conv_gather_outputs, causal_conv_slice_inputs +from .global_config import set_norm_limit +from .types import CausalAutoencoderOutput, CausalDecoderOutput, CausalEncoderOutput, MemoryState, _inflation_mode_t, _memory_device_t, _receptive_field_t + + +logger = get_logger(__name__) # pylint: disable=invalid-name + + +class Upsample3D(Upsample2D): + """A 3D upsampling layer with an optional convolution.""" + + def __init__( + self, + *args, + inflation_mode: _inflation_mode_t = "tail", + temporal_up: bool = False, + spatial_up: bool = True, + slicing: bool = False, + **kwargs, + ): + super().__init__(*args, **kwargs) + conv = self.conv if self.name == "conv" else self.Conv2d_0 + + assert type(conv) is not nn.ConvTranspose2d + # Note: lora_layer is not passed into constructor in the original implementation. + # So we make a simplification. + conv = init_causal_conv3d( + self.channels, + self.out_channels, + 3, + padding=1, + inflation_mode=inflation_mode, + ) + + self.temporal_up = temporal_up + self.spatial_up = spatial_up + self.temporal_ratio = 2 if temporal_up else 1 + self.spatial_ratio = 2 if spatial_up else 1 + self.slicing = slicing + + assert not self.interpolate + # [Override] MAGViT v2 implementation + if not self.interpolate: + upscale_ratio = (self.spatial_ratio**2) * self.temporal_ratio + self.upscale_conv = nn.Conv3d( + self.channels, self.channels * upscale_ratio, kernel_size=1, padding=0 + ) + identity = ( + torch.eye(self.channels) + .repeat(upscale_ratio, 1) + .reshape_as(self.upscale_conv.weight) + ) + self.upscale_conv.weight.data.copy_(identity) + nn.init.zeros_(self.upscale_conv.bias) + + if self.name == "conv": + self.conv = conv + else: + self.Conv2d_0 = conv + + def forward( + self, + hidden_states: torch.FloatTensor, + output_size: Optional[int] = None, + memory_state: MemoryState = MemoryState.DISABLED, + **kwargs, + ) -> torch.FloatTensor: + assert hidden_states.shape[1] == self.channels + + if hasattr(self, "norm") and self.norm is not None: + # [Overridden] change to causal norm. + hidden_states = causal_norm_wrapper(self.norm, hidden_states) + + if self.use_conv_transpose: + return self.conv(hidden_states) + + if self.slicing: + split_size = hidden_states.size(2) // 2 + hidden_states = list( + hidden_states.split([split_size, hidden_states.size(2) - split_size], dim=2) + ) + else: + hidden_states = [hidden_states] + # ADD BY NUMZ + for i in range(len(hidden_states)): + hidden_states[i] = self.upscale_conv(hidden_states[i]) + hidden_states[i] = rearrange( + hidden_states[i], + "b (x y z c) f h w -> b c (f z) (h x) (w y)", + x=self.spatial_ratio, + y=self.spatial_ratio, + z=self.temporal_ratio, + ) + + # [Overridden] For causal temporal conv + if self.temporal_up and memory_state != MemoryState.ACTIVE: + hidden_states[0] = remove_head(hidden_states[0]) + + if not self.slicing: + hidden_states = hidden_states[0] + # ADD BY NUMZ + if self.use_conv: + if self.name == "conv": + hidden_states = self.conv(hidden_states, memory_state=memory_state) + else: + hidden_states = self.Conv2d_0(hidden_states, memory_state=memory_state) + + if not self.slicing: + return hidden_states + else: + return torch.cat(hidden_states, dim=2) + + +class Downsample3D(Downsample2D): + """A 3D downsampling layer with an optional convolution.""" + + def __init__( + self, + *args, + inflation_mode: _inflation_mode_t = "tail", + spatial_down: bool = False, + temporal_down: bool = False, + **kwargs, + ): + super().__init__(*args, **kwargs) + conv = self.conv + self.temporal_down = temporal_down + self.spatial_down = spatial_down + + self.temporal_ratio = 2 if temporal_down else 1 + self.spatial_ratio = 2 if spatial_down else 1 + + self.temporal_kernel = 3 if temporal_down else 1 + self.spatial_kernel = 3 if spatial_down else 1 + + if type(conv) in [nn.Conv2d, LoRACompatibleConv]: + # Note: lora_layer is not passed into constructor in the original implementation. + # So we make a simplification. + conv = init_causal_conv3d( + self.channels, + self.out_channels, + kernel_size=(self.temporal_kernel, self.spatial_kernel, self.spatial_kernel), + stride=(self.temporal_ratio, self.spatial_ratio, self.spatial_ratio), + padding=( + 1 if self.temporal_down else 0, + self.padding if self.spatial_down else 0, + self.padding if self.spatial_down else 0, + ), + inflation_mode=inflation_mode, + ) + elif type(conv) is nn.AvgPool2d: + assert self.channels == self.out_channels + conv = nn.AvgPool3d( + kernel_size=(self.temporal_ratio, self.spatial_ratio, self.spatial_ratio), + stride=(self.temporal_ratio, self.spatial_ratio, self.spatial_ratio), + ) + else: + raise NotImplementedError + + if self.name == "conv": + self.Conv2d_0 = conv + self.conv = conv + else: + self.conv = conv + + def forward( + self, + hidden_states: torch.FloatTensor, + memory_state: MemoryState = MemoryState.DISABLED, + **kwargs, + ) -> torch.FloatTensor: + + assert hidden_states.shape[1] == self.channels + + if hasattr(self, "norm") and self.norm is not None: + # [Overridden] change to causal norm. + hidden_states = causal_norm_wrapper(self.norm, hidden_states) + + if self.use_conv and self.padding == 0 and self.spatial_down: + pad = (0, 1, 0, 1) + hidden_states = safe_pad_operation(hidden_states, pad, mode="constant", value=0) + + assert hidden_states.shape[1] == self.channels + + hidden_states = self.conv(hidden_states, memory_state=memory_state) + + return hidden_states + + +class ResnetBlock3D(ResnetBlock2D): + def __init__( + self, + *args, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + slicing: bool = False, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.conv1 = init_causal_conv3d( + self.in_channels, + self.out_channels, + kernel_size=(1, 3, 3) if time_receptive_field == "half" else (3, 3, 3), + stride=1, + padding=(0, 1, 1) if time_receptive_field == "half" else (1, 1, 1), + inflation_mode=inflation_mode, + ) + + self.conv2 = init_causal_conv3d( + self.out_channels, + self.conv2.out_channels, + kernel_size=3, + stride=1, + padding=1, + inflation_mode=inflation_mode, + ) + + if self.up: + assert type(self.upsample) is Upsample2D + self.upsample = Upsample3D( + self.in_channels, + use_conv=False, + inflation_mode=inflation_mode, + slicing=slicing, + ) + elif self.down: + assert type(self.downsample) is Downsample2D + self.downsample = Downsample3D( + self.in_channels, + use_conv=False, + padding=1, + name="op", + inflation_mode=inflation_mode, + ) + + if self.use_in_shortcut: + self.conv_shortcut = init_causal_conv3d( + self.in_channels, + self.conv_shortcut.out_channels, + kernel_size=1, + stride=1, + padding=0, + bias=(self.conv_shortcut.bias is not None), + inflation_mode=inflation_mode, + ) + + def forward( + self, input_tensor, temb, memory_state: MemoryState = MemoryState.DISABLED, **kwargs + ): + hidden_states = input_tensor + + hidden_states = causal_norm_wrapper(self.norm1, hidden_states) + hidden_states = self.nonlinearity(hidden_states) + + if self.upsample is not None: + # upsample_nearest_nhwc fails with large batch sizes. + # see https://github.com/huggingface/diffusers/issues/984 + if hidden_states.shape[0] >= 64: + input_tensor = input_tensor.contiguous() + hidden_states = hidden_states.contiguous() + input_tensor = self.upsample(input_tensor, memory_state=memory_state) + hidden_states = self.upsample(hidden_states, memory_state=memory_state) + elif self.downsample is not None: + input_tensor = self.downsample(input_tensor, memory_state=memory_state) + hidden_states = self.downsample(hidden_states, memory_state=memory_state) + + hidden_states = self.conv1(hidden_states, memory_state=memory_state) + + if self.time_emb_proj is not None: + if not self.skip_time_act: + temb = self.nonlinearity(temb) + temb = self.time_emb_proj(temb)[:, :, None, None] + + if temb is not None and self.time_embedding_norm == "default": + hidden_states = hidden_states + temb + + hidden_states = causal_norm_wrapper(self.norm2, hidden_states) + + if temb is not None and self.time_embedding_norm == "scale_shift": + scale, shift = torch.chunk(temb, 2, dim=1) + hidden_states = hidden_states * (1 + scale) + shift + + hidden_states = self.nonlinearity(hidden_states) + + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states, memory_state=memory_state) + + if self.conv_shortcut is not None: + input_tensor = self.conv_shortcut(input_tensor, memory_state=memory_state) + + output_tensor = (input_tensor + hidden_states) / self.output_scale_factor + + return output_tensor + + +class DownEncoderBlock3D(DownEncoderBlock2D): + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + downsample_padding: int = 1, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + temporal_down: bool = True, + spatial_down: bool = True, + ): + super().__init__( + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + num_layers=num_layers, + resnet_eps=resnet_eps, + resnet_time_scale_shift=resnet_time_scale_shift, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_pre_norm=resnet_pre_norm, + output_scale_factor=output_scale_factor, + add_downsample=add_downsample, + downsample_padding=downsample_padding, + ) + resnets = [] + temporal_modules = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + # [Override] Replace module. + ResnetBlock3D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + ) + temporal_modules.append(nn.Identity()) + + self.resnets = nn.ModuleList(resnets) + self.temporal_modules = nn.ModuleList(temporal_modules) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + # [Override] Replace module. + Downsample3D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name="op", + temporal_down=temporal_down, + spatial_down=spatial_down, + inflation_mode=inflation_mode, + ) + ] + ) + else: + self.downsamplers = None + + def forward( + self, + hidden_states: torch.FloatTensor, + memory_state: MemoryState = MemoryState.DISABLED, + **kwargs, + ) -> torch.FloatTensor: + for resnet, temporal in zip(self.resnets, self.temporal_modules): + hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state) + hidden_states = temporal(hidden_states) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, memory_state=memory_state) + + return hidden_states + + +class UpDecoderBlock3D(UpDecoderBlock2D): + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", # default, spatial + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + temb_channels: Optional[int] = None, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + temporal_up: bool = True, + spatial_up: bool = True, + slicing: bool = False, + ): + super().__init__( + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + num_layers=num_layers, + resnet_eps=resnet_eps, + resnet_time_scale_shift=resnet_time_scale_shift, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_pre_norm=resnet_pre_norm, + output_scale_factor=output_scale_factor, + add_upsample=add_upsample, + temb_channels=temb_channels, + ) + resnets = [] + temporal_modules = [] + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + resnets.append( + # [Override] Replace module. + ResnetBlock3D( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + slicing=slicing, + ) + ) + + temporal_modules.append(nn.Identity()) + + self.resnets = nn.ModuleList(resnets) + self.temporal_modules = nn.ModuleList(temporal_modules) + + if add_upsample: + # [Override] Replace module & use learnable upsample + self.upsamplers = nn.ModuleList( + [ + Upsample3D( + out_channels, + use_conv=True, + out_channels=out_channels, + temporal_up=temporal_up, + spatial_up=spatial_up, + interpolate=False, + inflation_mode=inflation_mode, + slicing=slicing, + ) + ] + ) + else: + self.upsamplers = None + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + memory_state: MemoryState = MemoryState.DISABLED, + ) -> torch.FloatTensor: + for resnet, temporal in zip(self.resnets, self.temporal_modules): + hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state) + hidden_states = temporal(hidden_states) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, memory_state=memory_state) + + return hidden_states + + +class UNetMidBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", # default, spatial + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + add_attention: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + ): + super().__init__() + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + self.add_attention = add_attention + + # there is always at least one resnet + resnets = [ + # [Override] Replace module. + ResnetBlock3D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + ] + attentions = [] + + if attention_head_dim is None: + logger.warn( + f"It is not recommend to pass `attention_head_dim=None`. " + f"Defaulting `attention_head_dim` to `in_channels`: {in_channels}." + ) + attention_head_dim = in_channels + + for _ in range(num_layers): + if self.add_attention: + attentions.append( + Attention( + in_channels, + heads=in_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=( + resnet_groups if resnet_time_scale_shift == "default" else None + ), + spatial_norm_dim=( + temb_channels if resnet_time_scale_shift == "spatial" else None + ), + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + else: + attentions.append(None) + + resnets.append( + ResnetBlock3D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward(self, hidden_states, temb=None, memory_state: MemoryState = MemoryState.DISABLED): + video_length, frame_height, frame_width = hidden_states.size()[-3:] + hidden_states = self.resnets[0](hidden_states, temb, memory_state=memory_state) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if attn is not None: + hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w") + hidden_states = attn(hidden_states, temb=temb) + hidden_states = rearrange( + hidden_states, "(b f) c h w -> b c f h w", f=video_length + ) + hidden_states = resnet(hidden_states, temb, memory_state=memory_state) + + return hidden_states + + +class Encoder3D(nn.Module): + r""" + [Override] override most logics to support extra condition input and causal conv + + The `Encoder` layer of a variational autoencoder that encodes + its input into a latent representation. + + Args: + in_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_channels (`int`, *optional*, defaults to 3): + The number of output channels. + down_block_types (`Tuple[str, ...]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + The types of down blocks to use. + See `~diffusers.models.unet_2d_blocks.get_down_block` + for available options. + block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + The number of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): + The number of layers per block. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups for normalization. + act_fn (`str`, *optional*, defaults to `"silu"`): + The activation function to use. + See `~diffusers.models.activations.get_activation` for available options. + double_z (`bool`, *optional*, defaults to `True`): + Whether to double the number of output channels for the last block. + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str, ...] = ("DownEncoderBlock3D",), + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 2, + norm_num_groups: int = 32, + act_fn: str = "silu", + double_z: bool = True, + mid_block_add_attention=True, + # [Override] add extra_cond_dim, temporal down num + temporal_down_num: int = 2, + extra_cond_dim: int = None, + gradient_checkpoint: bool = False, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + ): + super().__init__() + self.layers_per_block = layers_per_block + self.temporal_down_num = temporal_down_num + + self.conv_in = init_causal_conv3d( + in_channels, + block_out_channels[0], + kernel_size=3, + stride=1, + padding=1, + inflation_mode=inflation_mode, + ) + + self.mid_block = None + self.down_blocks = nn.ModuleList([]) + self.extra_cond_dim = extra_cond_dim + + self.conv_extra_cond = nn.ModuleList([]) + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + # [Override] to support temporal down block design + is_temporal_down_block = i >= len(block_out_channels) - self.temporal_down_num - 1 + # Note: take the last ones + + assert down_block_type == "DownEncoderBlock3D" + + down_block = DownEncoderBlock3D( + num_layers=self.layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + add_downsample=not is_final_block, + resnet_eps=1e-6, + downsample_padding=0, + # Note: Don't know why set it as 0 + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + temporal_down=is_temporal_down_block, + spatial_down=True, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + self.down_blocks.append(down_block) + + def zero_module(module): + # Zero out the parameters of a module and return it. + for p in module.parameters(): + p.detach().zero_() + return module + + self.conv_extra_cond.append( + zero_module( + nn.Conv3d(extra_cond_dim, output_channel, kernel_size=1, stride=1, padding=0) + ) + if self.extra_cond_dim is not None and self.extra_cond_dim > 0 + else None + ) + + # mid + self.mid_block = UNetMidBlock3D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + output_scale_factor=1, + resnet_time_scale_shift="default", + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + temb_channels=None, + add_attention=mid_block_add_attention, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + + # out + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6 + ) + self.conv_act = nn.SiLU() + + conv_out_channels = 2 * out_channels if double_z else out_channels + self.conv_out = init_causal_conv3d( + block_out_channels[-1], conv_out_channels, 3, padding=1, inflation_mode=inflation_mode + ) + + self.gradient_checkpointing = gradient_checkpoint + + def forward( + self, + sample: torch.FloatTensor, + extra_cond=None, + memory_state: MemoryState = MemoryState.DISABLED, + ) -> torch.FloatTensor: + r"""The forward method of the `Encoder` class.""" + sample = self.conv_in(sample, memory_state=memory_state) + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + # down + # [Override] add extra block and extra cond + for down_block, extra_block in zip(self.down_blocks, self.conv_extra_cond): + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(down_block), sample, memory_state, use_reentrant=False + ) + if extra_block is not None: + sample = sample + safe_interpolate_operation(extra_block(extra_cond), size=sample.shape[2:]) + + # middle + sample = self.mid_block(sample, memory_state=memory_state) + + # sample = torch.utils.checkpoint.checkpoint( + # create_custom_forward(self.mid_block), sample, use_reentrant=False + # ) + + else: + # down + # [Override] add extra block and extra cond + for down_block, extra_block in zip(self.down_blocks, self.conv_extra_cond): + sample = down_block(sample, memory_state=memory_state) + if extra_block is not None: + sample = sample + safe_interpolate_operation(extra_block(extra_cond), size=sample.shape[2:]) + + # middle + sample = self.mid_block(sample, memory_state=memory_state) + + # post-process + sample = causal_norm_wrapper(self.conv_norm_out, sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample, memory_state=memory_state) + + return sample + + +class Decoder3D(nn.Module): + r""" + The `Decoder` layer of a variational autoencoder that + decodes its latent representation into an output sample. + + Args: + in_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_channels (`int`, *optional*, defaults to 3): + The number of output channels. + up_block_types (`Tuple[str, ...]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + The types of up blocks to use. + See `~diffusers.models.unet_2d_blocks.get_up_block` for available options. + block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + The number of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): + The number of layers per block. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups for normalization. + act_fn (`str`, *optional*, defaults to `"silu"`): + The activation function to use. + See `~diffusers.models.activations.get_activation` for available options. + norm_type (`str`, *optional*, defaults to `"group"`): + The normalization type to use. Can be either `"group"` or `"spatial"`. + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + up_block_types: Tuple[str, ...] = ("UpDecoderBlock3D",), + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 2, + norm_num_groups: int = 32, + act_fn: str = "silu", + norm_type: str = "group", # group, spatial + mid_block_add_attention=True, + # [Override] add temporal up block + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + temporal_up_num: int = 2, + slicing_up_num: int = 0, + gradient_checkpoint: bool = False, + ): + super().__init__() + self.layers_per_block = layers_per_block + self.temporal_up_num = temporal_up_num + + self.conv_in = init_causal_conv3d( + in_channels, + block_out_channels[-1], + kernel_size=3, + stride=1, + padding=1, + inflation_mode=inflation_mode, + ) + + self.mid_block = None + self.up_blocks = nn.ModuleList([]) + + temb_channels = in_channels if norm_type == "spatial" else None + + # mid + self.mid_block = UNetMidBlock3D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + output_scale_factor=1, + resnet_time_scale_shift="default" if norm_type == "group" else norm_type, + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + temb_channels=temb_channels, + add_attention=mid_block_add_attention, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + + is_final_block = i == len(block_out_channels) - 1 + is_temporal_up_block = i < self.temporal_up_num + is_slicing_up_block = i >= len(block_out_channels) - slicing_up_num + # Note: Keep symmetric + + assert up_block_type == "UpDecoderBlock3D" + up_block = UpDecoderBlock3D( + num_layers=self.layers_per_block + 1, + in_channels=prev_output_channel, + out_channels=output_channel, + add_upsample=not is_final_block, + resnet_eps=1e-6, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + resnet_time_scale_shift=norm_type, + temb_channels=temb_channels, + temporal_up=is_temporal_up_block, + slicing=is_slicing_up_block, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + if norm_type == "spatial": + self.conv_norm_out = SpatialNorm(block_out_channels[0], temb_channels) + else: + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6 + ) + self.conv_act = nn.SiLU() + self.conv_out = init_causal_conv3d( + block_out_channels[0], out_channels, 3, padding=1, inflation_mode=inflation_mode + ) + + self.gradient_checkpointing = gradient_checkpoint + + # Note: Just copy from Decoder. + def forward( + self, + sample: torch.FloatTensor, + latent_embeds: Optional[torch.FloatTensor] = None, + memory_state: MemoryState = MemoryState.DISABLED, + ) -> torch.FloatTensor: + r"""The forward method of the `Decoder` class.""" + + sample = self.conv_in(sample, memory_state=memory_state) + + #upscale_dtype = next(iter(self.up_blocks.parameters())).dtype + upscale_dtype = sample.dtype + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + sample = self.mid_block(sample, latent_embeds, memory_state=memory_state) + sample = sample.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(up_block), + sample, + latent_embeds, + memory_state, + use_reentrant=False, + ) + else: + # middle + sample = self.mid_block(sample, latent_embeds, memory_state=memory_state) + sample = sample.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(up_block), sample, latent_embeds, memory_state + ) + else: + # middle + sample = self.mid_block(sample, latent_embeds, memory_state=memory_state) + sample = sample.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + sample = up_block(sample, latent_embeds, memory_state=memory_state) + + # post-process + sample = causal_norm_wrapper(self.conv_norm_out, sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample, memory_state=memory_state) + + return sample + + +class AutoencoderKL(diffusers.AutoencoderKL): + """ + We simply inherit the model code from diffusers + """ + + def __init__(self, attention: bool = True, *args, **kwargs): + super().__init__(*args, **kwargs) + + # A hacky way to remove attention. + if not attention: + self.encoder.mid_block.attentions = torch.nn.ModuleList([None]) + self.decoder.mid_block.attentions = torch.nn.ModuleList([None]) + + def load_state_dict(self, state_dict, strict=True): + # Newer version of diffusers changed the model keys, + # causing incompatibility with old checkpoints. + # They provided a method for conversion. We call conversion before loading state_dict. + convert_deprecated_attention_blocks = getattr( + self, "_convert_deprecated_attention_blocks", None + ) + if callable(convert_deprecated_attention_blocks): + convert_deprecated_attention_blocks(state_dict) + return super().load_state_dict(state_dict, strict) + + +class VideoAutoencoderKL(diffusers.AutoencoderKL): + """ + We simply inherit the model code from diffusers + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str] = ("DownEncoderBlock3D",), + up_block_types: Tuple[str] = ("UpDecoderBlock3D",), + block_out_channels: Tuple[int] = (64,), + layers_per_block: int = 1, + act_fn: str = "silu", + latent_channels: int = 4, + norm_num_groups: int = 32, + sample_size: int = 32, + scaling_factor: float = 0.18215, + force_upcast: float = True, + attention: bool = True, + temporal_scale_num: int = 0, + slicing_up_num: int = 0, + gradient_checkpoint: bool = False, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "full", + slicing_sample_min_size: int = 32, + use_quant_conv: bool = True, + use_post_quant_conv: bool = True, + *args, + **kwargs, + ): + extra_cond_dim = kwargs.pop("extra_cond_dim") if "extra_cond_dim" in kwargs else None + self.slicing_sample_min_size = slicing_sample_min_size + self.slicing_latent_min_size = slicing_sample_min_size // (2**temporal_scale_num) + + super().__init__( + in_channels=in_channels, + out_channels=out_channels, + # [Override] make sure it can be normally initialized + down_block_types=tuple( + [down_block_type.replace("3D", "2D") for down_block_type in down_block_types] + ), + up_block_types=tuple( + [up_block_type.replace("3D", "2D") for up_block_type in up_block_types] + ), + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + act_fn=act_fn, + latent_channels=latent_channels, + norm_num_groups=norm_num_groups, + sample_size=sample_size, + scaling_factor=scaling_factor, + force_upcast=force_upcast, + *args, + **kwargs, + ) + + # pass init params to Encoder + self.encoder = Encoder3D( + in_channels=in_channels, + out_channels=latent_channels, + down_block_types=down_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + act_fn=act_fn, + norm_num_groups=norm_num_groups, + double_z=True, + extra_cond_dim=extra_cond_dim, + # [Override] add temporal_down_num parameter + temporal_down_num=temporal_scale_num, + gradient_checkpoint=gradient_checkpoint, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + + # pass init params to Decoder + self.decoder = Decoder3D( + in_channels=latent_channels, + out_channels=out_channels, + up_block_types=up_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + norm_num_groups=norm_num_groups, + act_fn=act_fn, + # [Override] add temporal_up_num parameter + temporal_up_num=temporal_scale_num, + slicing_up_num=slicing_up_num, + gradient_checkpoint=gradient_checkpoint, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + + self.quant_conv = ( + init_causal_conv3d( + in_channels=2 * latent_channels, + out_channels=2 * latent_channels, + kernel_size=1, + inflation_mode=inflation_mode, + ) + if use_quant_conv + else None + ) + self.post_quant_conv = ( + init_causal_conv3d( + in_channels=latent_channels, + out_channels=latent_channels, + kernel_size=1, + inflation_mode=inflation_mode, + ) + if use_post_quant_conv + else None + ) + + # A hacky way to remove attention. + if not attention: + self.encoder.mid_block.attentions = torch.nn.ModuleList([None]) + self.decoder.mid_block.attentions = torch.nn.ModuleList([None]) + + @apply_forward_hook + def encode(self, x: torch.FloatTensor, return_dict: bool = True) -> AutoencoderKLOutput: + # h = self.slicing_encode(x) + h = self.tiled_encode(x) + posterior = DiagonalGaussianDistribution(h) + + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + @apply_forward_hook + def decode( + self, z: torch.Tensor, return_dict: bool = True + ) -> Union[DecoderOutput, torch.Tensor]: + # decoded = self.slicing_decode(z) + decoded = self.tiled_decode(z) + + if not return_dict: + return (decoded,) + + return DecoderOutput(sample=decoded) + + def _encode( + self, x: torch.Tensor, memory_state: MemoryState = MemoryState.DISABLED + ) -> torch.Tensor: + _x = x.to(self.device) + _x = causal_conv_slice_inputs(_x, self.slicing_sample_min_size, memory_state=memory_state) + h = self.encoder(_x, memory_state=memory_state) + if self.quant_conv is not None: + output = self.quant_conv(h, memory_state=memory_state) + else: + output = h + output = causal_conv_gather_outputs(output) + return output.to(x.device) + + def _decode( + self, z: torch.Tensor, memory_state: MemoryState = MemoryState.DISABLED + ) -> torch.Tensor: + _z = z.to(self.device) + _z = causal_conv_slice_inputs(_z, self.slicing_latent_min_size, memory_state=memory_state) + if self.post_quant_conv is not None: + _z = self.post_quant_conv(_z, memory_state=memory_state) + output = self.decoder(_z, memory_state=memory_state) + output = causal_conv_gather_outputs(output) + return output.to(z.device) + + def slicing_encode(self, x: torch.Tensor) -> torch.Tensor: + sp_size = 1 + if self.use_slicing and (x.shape[2] - 1) > self.slicing_sample_min_size * sp_size: + x_slices = x[:, :, 1:].split(split_size=self.slicing_sample_min_size * sp_size, dim=2) + encoded_slices = [ + self._encode( + torch.cat((x[:, :, :1], x_slices[0]), dim=2), + memory_state=MemoryState.INITIALIZING, + ) + ] + for x_idx in range(1, len(x_slices)): + encoded_slices.append( + self._encode(x_slices[x_idx], memory_state=MemoryState.ACTIVE) + ) + return torch.cat(encoded_slices, dim=2) + else: + return self._encode(x) + + def slicing_decode(self, z: torch.Tensor) -> torch.Tensor: + sp_size = 1 + if self.use_slicing and (z.shape[2] - 1) > self.slicing_latent_min_size * sp_size: + z_slices = z[:, :, 1:].split(split_size=self.slicing_latent_min_size * sp_size, dim=2) + decoded_slices = [ + self._decode( + torch.cat((z[:, :, :1], z_slices[0]), dim=2), + memory_state=MemoryState.INITIALIZING, + ) + ] + for z_idx in range(1, len(z_slices)): + decoded_slices.append( + self._decode(z_slices[z_idx], memory_state=MemoryState.ACTIVE) + ) + return torch.cat(decoded_slices, dim=2) + else: + return self._decode(z) + + def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[3], b.shape[3], blend_extent) + for y in range(blend_extent): + b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * (y / blend_extent) + return b + + def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[4], b.shape[4], blend_extent) + for x in range(blend_extent): + b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * (x / blend_extent) + return b + + def tiled_encode(self, x: torch.Tensor) -> torch.Tensor: + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = self.tile_latent_min_size - blend_extent + rows = [] + for i in range(0, x.shape[3], overlap_size): + row = [] + for j in range(0, x.shape[4], overlap_size): + tile = x[:, :, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size] + tile = self._encode(tile) + row.append(tile) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=4)) + enc = torch.cat(result_rows, dim=3) + return enc + + def tiled_decode(self, z: torch.Tensor) -> torch.Tensor: + overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor) + row_limit = self.tile_sample_min_size - blend_extent + rows = [] + for i in range(0, z.shape[3], overlap_size): + row = [] + for j in range(0, z.shape[4], overlap_size): + tile = z[:, :, :, i : i + self.tile_latent_min_size, j : j + self.tile_latent_min_size] + decoded = self.decoder(tile) + row.append(decoded) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=4)) + dec = torch.cat(result_rows, dim=3) + return dec + + def forward( + self, x: torch.FloatTensor, mode: Literal["encode", "decode", "all"] = "all", **kwargs + ): + # x: [b c t h w] + if mode == "encode": + h = self.encode(x) + return h.latent_dist + elif mode == "decode": + h = self.decode(x) + return h.sample + else: + h = self.encode(x) + h = self.decode(h.latent_dist.mode()) + return h.sample + + def load_state_dict(self, state_dict, strict=False): + # Newer version of diffusers changed the model keys, + # causing incompatibility with old checkpoints. + # They provided a method for conversion. + # We call conversion before loading state_dict. + convert_deprecated_attention_blocks = getattr( + self, "_convert_deprecated_attention_blocks", None + ) + if callable(convert_deprecated_attention_blocks): + convert_deprecated_attention_blocks(state_dict) + return super().load_state_dict(state_dict, strict) + + +class VideoAutoencoderKLWrapper(VideoAutoencoderKL): + def __init__( + self, + *args, + spatial_downsample_factor: int, + temporal_downsample_factor: int, + freeze_encoder: bool, + **kwargs, + ): + self.spatial_downsample_factor = spatial_downsample_factor + self.temporal_downsample_factor = temporal_downsample_factor + self.freeze_encoder = freeze_encoder + super().__init__(*args, **kwargs) + + def forward(self, x: torch.FloatTensor) -> CausalAutoencoderOutput: + with torch.no_grad() if self.freeze_encoder else nullcontext(): + z, p = self.encode(x) + x = self.decode(z).sample + return CausalAutoencoderOutput(x, z, p) + + def encode(self, x: torch.FloatTensor) -> CausalEncoderOutput: + if x.ndim == 4: + x = x.unsqueeze(2) + p = super().encode(x).latent_dist + z = p.sample().squeeze(2) + return CausalEncoderOutput(z, p) + + def decode(self, z: torch.FloatTensor) -> CausalDecoderOutput: + if z.ndim == 4: + z = z.unsqueeze(2) + x = super().decode(z).sample.squeeze(2) + return CausalDecoderOutput(x) + + def preprocess(self, x: torch.Tensor): + # x should in [B, C, T, H, W], [B, C, H, W] + assert x.ndim == 4 or x.size(2) % 4 == 1 + return x + + def postprocess(self, x: torch.Tensor): + # x should in [B, C, T, H, W], [B, C, H, W] + return x + + def set_causal_slicing( + self, + *, + split_size: Optional[int], + memory_device: _memory_device_t, + ): + assert ( + split_size is None or memory_device is not None + ), "if split_size is set, memory_device must not be None." + if split_size is not None: + self.enable_slicing() + self.slicing_sample_min_size = split_size + self.slicing_latent_min_size = split_size // self.temporal_downsample_factor + else: + self.disable_slicing() + for module in self.modules(): + if isinstance(module, InflatedCausalConv3d): + module.set_memory_device(memory_device) + + def set_memory_limit(self, conv_max_mem: Optional[float], norm_max_mem: Optional[float]): + set_norm_limit(norm_max_mem) + for m in self.modules(): + if isinstance(m, InflatedCausalConv3d): + m.set_memory_limit(conv_max_mem if conv_max_mem is not None else float("inf")) diff --git a/modules/seedvr/src/models/video_vae_v3/modules/causal_inflation_lib.py b/modules/seedvr/src/models/video_vae_v3/modules/causal_inflation_lib.py new file mode 100644 index 000000000..c6d35f0cb --- /dev/null +++ b/modules/seedvr/src/models/video_vae_v3/modules/causal_inflation_lib.py @@ -0,0 +1,417 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +import math +from contextlib import contextmanager +from typing import List, Optional, Union +import torch +import torch.nn.functional as F +from diffusers.models.normalization import RMSNorm +from einops import rearrange +from torch import Tensor, nn +from torch.nn import Conv3d +from .context_parallel_lib import cache_send_recv, get_cache_size +from .global_config import get_norm_limit +from .types import MemoryState, _inflation_mode_t, _memory_device_t +from ....common.half_precision_fixes import safe_pad_operation + + +@contextmanager +def ignore_padding(model): + orig_padding = model.padding + model.padding = (0, 0, 0) + try: + yield + finally: + model.padding = orig_padding + + +class InflatedCausalConv3d(Conv3d): + def __init__( + self, + *args, + inflation_mode: _inflation_mode_t, + memory_device: _memory_device_t = "same", + **kwargs, + ): + self.inflation_mode = inflation_mode + self.memory = None + super().__init__(*args, **kwargs) + self.temporal_padding = self.padding[0] + self.memory_device = memory_device + self.padding = (0, *self.padding[1:]) # Remove temporal pad to keep causal. + self.memory_limit = float("inf") + + def set_memory_limit(self, value: float): + self.memory_limit = value + + def set_memory_device(self, memory_device: _memory_device_t): + self.memory_device = memory_device + + def memory_limit_conv( + self, + x, + *, + split_dim=3, + padding=(0, 0, 0, 0, 0, 0), + prev_cache=None, + ): + # Compatible with no limit. + if math.isinf(self.memory_limit): + if prev_cache is not None: + x = torch.cat([prev_cache, x], dim=split_dim - 1) + return super().forward(x) + + # Compute tensor shape after concat & padding. + shape = torch.tensor(x.size()) + if prev_cache is not None: + shape[split_dim - 1] += prev_cache.size(split_dim - 1) + shape[-3:] += torch.tensor(padding).view(3, 2).sum(-1).flip(0) + memory_occupy = shape.prod() * x.element_size() / 1024**3 # GiB + if memory_occupy < self.memory_limit or split_dim == x.ndim: + if prev_cache is not None: + x = torch.cat([prev_cache, x], dim=split_dim - 1) + x = safe_pad_operation(x, padding, mode='constant', value=0.0) + with ignore_padding(self): + return super().forward(x) + + # Exceed memory limit, splitting tensor + + # Split input (& prev_cache). + num_splits = math.ceil(memory_occupy / self.memory_limit) + size_per_split = x.size(split_dim) // num_splits + split_sizes = [size_per_split] * (num_splits - 1) + split_sizes += [x.size(split_dim) - sum(split_sizes)] + + x = list(x.split(split_sizes, dim=split_dim)) + if prev_cache is not None: + prev_cache = list(prev_cache.split(split_sizes, dim=split_dim)) + # Loop Fwd. + cache = None + for idx in range(len(x)): + # Concat prev cache from last dim + if prev_cache is not None: + x[idx] = torch.cat([prev_cache[idx], x[idx]], dim=split_dim - 1) + + # Get padding pattern. + lpad_dim = (x[idx].ndim - split_dim - 1) * 2 + rpad_dim = lpad_dim + 1 + padding = list(padding) + padding[lpad_dim] = self.padding[split_dim - 2] if idx == 0 else 0 + padding[rpad_dim] = self.padding[split_dim - 2] if idx == len(x) - 1 else 0 + pad_len = padding[lpad_dim] + padding[rpad_dim] + padding = tuple(padding) + + # Prepare cache for next slice (this dim). + next_cache = None + cache_len = cache.size(split_dim) if cache is not None else 0 + next_catch_size = get_cache_size( + conv_module=self, + input_len=x[idx].size(split_dim) + cache_len, + pad_len=pad_len, + dim=split_dim - 2, + ) + if next_catch_size != 0: + assert next_catch_size <= x[idx].size(split_dim) + next_cache = ( + x[idx].transpose(0, split_dim)[-next_catch_size:].transpose(0, split_dim) + ) + + # Recursive. + x[idx] = self.memory_limit_conv( + x[idx], + split_dim=split_dim + 1, + padding=padding, + prev_cache=cache, + ) + + # Update cache. + cache = next_cache + + output = torch.cat(x, split_dim) + return output + + def forward( + self, + input: Union[Tensor, List[Tensor]], + memory_state: MemoryState = MemoryState.UNSET, + ) -> Tensor: + assert memory_state != MemoryState.UNSET + if memory_state != MemoryState.ACTIVE: + self.memory = None + if ( + math.isinf(self.memory_limit) + and torch.is_tensor(input) + ): + return self.basic_forward(input, memory_state) + return self.slicing_forward(input, memory_state) + + def basic_forward(self, input: Tensor, memory_state: MemoryState = MemoryState.UNSET): + mem_size = self.stride[0] - self.kernel_size[0] + if (self.memory is not None) and (memory_state == MemoryState.ACTIVE): + input = extend_head(input, memory=self.memory, times=-1) + else: + input = extend_head(input, times=self.temporal_padding * 2) + memory = ( + input[:, :, mem_size:].detach() + if (mem_size != 0 and memory_state != MemoryState.DISABLED) + else None + ) + if ( + memory_state != MemoryState.DISABLED + and not self.training + and (self.memory_device is not None) + ): + self.memory = memory + if self.memory_device == "cpu" and self.memory is not None: + self.memory = self.memory.to("cpu") + return super().forward(input) + + def slicing_forward( + self, + input: Union[Tensor, List[Tensor]], + memory_state: MemoryState = MemoryState.UNSET, + ) -> Tensor: + squeeze_out = False + if torch.is_tensor(input): + input = [input] + squeeze_out = True + + cache_size = self.kernel_size[0] - self.stride[0] + cache = cache_send_recv( + input, cache_size=cache_size, memory=self.memory, times=self.temporal_padding * 2 + ) + + # Single GPU inference - simplified memory management + if ( + memory_state in [MemoryState.INITIALIZING, MemoryState.ACTIVE] # use_slicing + and not self.training + and (self.memory_device is not None) + and cache_size != 0 + ): + if cache_size > input[-1].size(2) and cache is not None and len(input) == 1: + input[0] = torch.cat([cache, input[0]], dim=2) + cache = None + if cache_size <= input[-1].size(2): + self.memory = input[-1][:, :, -cache_size:].detach().contiguous() + if self.memory_device == "cpu" and self.memory is not None: + self.memory = self.memory.to("cpu") + + padding = tuple(x for x in reversed(self.padding) for _ in range(2)) + for i in range(len(input)): + # Prepare cache for next input slice. + next_cache = None + cache_size = 0 + if i < len(input) - 1: + cache_len = cache.size(2) if cache is not None else 0 + cache_size = get_cache_size(self, input[i].size(2) + cache_len, pad_len=0) + if cache_size != 0: + if cache_size > input[i].size(2) and cache is not None: + input[i] = torch.cat([cache, input[i]], dim=2) + cache = None + assert cache_size <= input[i].size(2), f"{cache_size} > {input[i].size(2)}" + next_cache = input[i][:, :, -cache_size:] + + # Conv forward for this input slice. + input[i] = self.memory_limit_conv( + input[i], + padding=padding, + prev_cache=cache, + ) + + # Update cache. + cache = next_cache + + return input[0] if squeeze_out else input + + def tflops(self, args, kwargs, output) -> float: + if torch.is_tensor(output): + output_numel = output.numel() + elif isinstance(output, list): + output_numel = sum(o.numel() for o in output) + else: + raise NotImplementedError + return (2 * math.prod(self.kernel_size) * self.in_channels * (output_numel / 1e6)) / 1e6 + + def _load_from_state_dict( + self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ): + if self.inflation_mode != "none": + state_dict = modify_state_dict( + self, + state_dict, + prefix, + inflate_weight_fn=inflate_weight, + inflate_bias_fn=inflate_bias, + ) + super()._load_from_state_dict( + state_dict, + prefix, + local_metadata, + (strict and self.inflation_mode == "none"), + missing_keys, + unexpected_keys, + error_msgs, + ) + + +def init_causal_conv3d( + *args, + inflation_mode: _inflation_mode_t, + **kwargs, +): + """ + Initialize a Causal-3D convolution layer. + Parameters: + inflation_mode: Listed as below. It's compatible with all the 3D-VAE checkpoints we have. + - none: No inflation will be conducted. + The loading logic of state dict will fall back to default. + - tail / replicate: Refer to the definition of `InflatedCausalConv3d`. + """ + return InflatedCausalConv3d(*args, inflation_mode=inflation_mode, **kwargs) + + +def causal_norm_wrapper(norm_layer: nn.Module, x: torch.Tensor) -> torch.Tensor: + input_dtype = x.dtype + if isinstance(norm_layer, (nn.LayerNorm, RMSNorm)): + if x.ndim == 4: + x = rearrange(x, "b c h w -> b h w c") + x = norm_layer(x) + x = rearrange(x, "b h w c -> b c h w") + return x.to(input_dtype) + if x.ndim == 5: + x = rearrange(x, "b c t h w -> b t h w c") + x = norm_layer(x) + x = rearrange(x, "b t h w c -> b c t h w") + return x.to(input_dtype) + if isinstance(norm_layer, (nn.GroupNorm, nn.BatchNorm2d, nn.SyncBatchNorm)): + if x.ndim <= 4: + return norm_layer(x).to(input_dtype) + if x.ndim == 5: + t = x.size(2) + x = rearrange(x, "b c t h w -> (b t) c h w") + memory_occupy = x.numel() * x.element_size() / 1024**3 + if isinstance(norm_layer, nn.GroupNorm) and memory_occupy > get_norm_limit(): + num_chunks = min(4 if x.element_size() == 2 else 2, norm_layer.num_groups) + assert norm_layer.num_groups % num_chunks == 0 + num_groups_per_chunk = norm_layer.num_groups // num_chunks + + x = list(x.chunk(num_chunks, dim=1)) + weights = norm_layer.weight.chunk(num_chunks, dim=0) + biases = norm_layer.bias.chunk(num_chunks, dim=0) + for i, (w, b) in enumerate(zip(weights, biases)): + x[i] = F.group_norm(x[i], num_groups_per_chunk, w, b, norm_layer.eps) + x[i] = x[i].to(input_dtype) + # ADD BY NUMZ + # ADD BY NUMZ + x = torch.cat(x, dim=1) + else: + x = norm_layer(x) + x = rearrange(x, "(b t) c h w -> b c t h w", t=t) + return x.to(input_dtype) + raise NotImplementedError + + +def remove_head(tensor: Tensor, times: int = 1) -> Tensor: + """ + Remove duplicated first frame features in the up-sampling process. + """ + # Single GPU inference - always process + if times == 0: + return tensor + return torch.cat(tensors=(tensor[:, :, :1], tensor[:, :, times + 1 :]), dim=2) + + +def extend_head(tensor: Tensor, times: int = 2, memory: Optional[Tensor] = None) -> Tensor: + """ + When memory is None: + - Duplicate first frame features in the down-sampling process. + When memory is not None: + - Concatenate memory features with the input features to keep temporal consistency. + """ + if memory is not None: + return torch.cat((memory.to(tensor), tensor), dim=2) + assert times >= 0, "Invalid input for function 'extend_head'!" + if times == 0: + return tensor + else: + tile_repeat = [1] * tensor.ndim + tile_repeat[2] = times + return torch.cat(tensors=(torch.tile(tensor[:, :, :1], tile_repeat), tensor), dim=2) + + +def inflate_weight(weight_2d: torch.Tensor, weight_3d: torch.Tensor, inflation_mode: str): + """ + Inflate a 2D convolution weight matrix to a 3D one. + Parameters: + weight_2d: The weight matrix of 2D conv to be inflated. + weight_3d: The weight matrix of 3D conv to be initialized. + inflation_mode: the mode of inflation + """ + assert inflation_mode in ["tail", "replicate"] + assert weight_3d.shape[:2] == weight_2d.shape[:2] + with torch.no_grad(): + if inflation_mode == "replicate": + depth = weight_3d.size(2) + weight_3d.copy_(weight_2d.unsqueeze(2).repeat(1, 1, depth, 1, 1) / depth) + else: + weight_3d.fill_(0.0) + weight_3d[:, :, -1].copy_(weight_2d) + return weight_3d + + +def inflate_bias(bias_2d: torch.Tensor, bias_3d: torch.Tensor, inflation_mode: str): + """ + Inflate a 2D convolution bias tensor to a 3D one + Parameters: + bias_2d: The bias tensor of 2D conv to be inflated. + bias_3d: The bias tensor of 3D conv to be initialized. + inflation_mode: Placeholder to align `inflate_weight`. + """ + assert bias_3d.shape == bias_2d.shape + with torch.no_grad(): + bias_3d.copy_(bias_2d) + return bias_3d + + +def modify_state_dict(layer, state_dict, prefix, inflate_weight_fn, inflate_bias_fn): + """ + the main function to inflated 2D parameters to 3D. + """ + weight_name = prefix + "weight" + bias_name = prefix + "bias" + if weight_name in state_dict: + weight_2d = state_dict[weight_name] + if weight_2d.dim() == 4: + # Assuming the 2D weights are 4D tensors (out_channels, in_channels, h, w) + weight_3d = inflate_weight_fn( + weight_2d=weight_2d, + weight_3d=layer.weight, + inflation_mode=layer.inflation_mode, + ) + state_dict[weight_name] = weight_3d + else: + return state_dict + # It's a 3d state dict, should not do inflation on both bias and weight. + if bias_name in state_dict: + bias_2d = state_dict[bias_name] + if bias_2d.dim() == 1: + # Assuming the 2D biases are 1D tensors (out_channels,) + bias_3d = inflate_bias_fn( + bias_2d=bias_2d, + bias_3d=layer.bias, + inflation_mode=layer.inflation_mode, + ) + state_dict[bias_name] = bias_3d + return state_dict diff --git a/modules/seedvr/src/models/video_vae_v3/modules/context_parallel_lib.py b/modules/seedvr/src/models/video_vae_v3/modules/context_parallel_lib.py new file mode 100644 index 000000000..830d18b87 --- /dev/null +++ b/modules/seedvr/src/models/video_vae_v3/modules/context_parallel_lib.py @@ -0,0 +1,63 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import List +import torch +from torch import Tensor + +# Single GPU inference - no distributed processing needed + + +def causal_conv_slice_inputs(x, split_size, memory_state): + # Single GPU inference - no slicing needed, return full tensor + return x + + +def causal_conv_gather_outputs(x): + # Single GPU inference - no gathering needed, return tensor as is + return x + + +def get_output_len(conv_module, input_len, pad_len, dim=0): + dilated_kernerl_size = conv_module.dilation[dim] * (conv_module.kernel_size[dim] - 1) + 1 + output_len = (input_len + pad_len - dilated_kernerl_size) // conv_module.stride[dim] + 1 + return output_len + + +def get_cache_size(conv_module, input_len, pad_len, dim=0): + dilated_kernerl_size = conv_module.dilation[dim] * (conv_module.kernel_size[dim] - 1) + 1 + output_len = (input_len + pad_len - dilated_kernerl_size) // conv_module.stride[dim] + 1 + remain_len = ( + input_len + pad_len - ((output_len - 1) * conv_module.stride[dim] + dilated_kernerl_size) + ) + overlap_len = dilated_kernerl_size - conv_module.stride[dim] + cache_len = overlap_len + remain_len # >= 0 + + assert output_len > 0 + return cache_len + + +def cache_send_recv(tensor: List[Tensor], cache_size, times, memory=None): + # Single GPU inference - simplified cache handling + recv_buffer = None + + # Handle memory buffer for single GPU case + if memory is not None: + recv_buffer = memory.to(tensor[0]) + elif times > 0: + tile_repeat = [1] * tensor[0].ndim + tile_repeat[2] = times + recv_buffer = torch.tile(tensor[0][:, :, :1], tile_repeat) + + return recv_buffer diff --git a/modules/seedvr/src/models/video_vae_v3/modules/global_config.py b/modules/seedvr/src/models/video_vae_v3/modules/global_config.py new file mode 100644 index 000000000..863117570 --- /dev/null +++ b/modules/seedvr/src/models/video_vae_v3/modules/global_config.py @@ -0,0 +1,28 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from typing import Optional + +_NORM_LIMIT = float("inf") + + +def get_norm_limit(): + return _NORM_LIMIT + + +def set_norm_limit(value: Optional[float] = None): + global _NORM_LIMIT + if value is None: + value = float("inf") + _NORM_LIMIT = value diff --git a/modules/seedvr/src/models/video_vae_v3/modules/inflated_layers.py b/modules/seedvr/src/models/video_vae_v3/modules/inflated_layers.py new file mode 100644 index 000000000..da8c1304f --- /dev/null +++ b/modules/seedvr/src/models/video_vae_v3/modules/inflated_layers.py @@ -0,0 +1,100 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from functools import partial +from typing import Literal, Optional +from torch import Tensor +from torch.nn import Conv3d +from .inflated_lib import MemoryState, extend_head, inflate_bias, inflate_weight, modify_state_dict + + +_inflation_mode_t = Literal["none", "tail", "replicate"] +_memory_device_t = Optional[Literal["cpu", "same"]] + + +class InflatedCausalConv3d(Conv3d): + def __init__( + self, + *args, + inflation_mode: _inflation_mode_t, + memory_device: _memory_device_t = "same", + **kwargs, + ): + self.inflation_mode = inflation_mode + self.memory = None + super().__init__(*args, **kwargs) + self.temporal_padding = self.padding[0] + self.memory_device = memory_device + self.padding = (0, *self.padding[1:]) # Remove temporal pad to keep causal. + + def set_memory_device(self, memory_device: _memory_device_t): + self.memory_device = memory_device + + def forward(self, input: Tensor, memory_state: MemoryState = MemoryState.DISABLED) -> Tensor: + mem_size = self.stride[0] - self.kernel_size[0] + if (self.memory is not None) and (memory_state == MemoryState.ACTIVE): + input = extend_head(input, memory=self.memory) + else: + input = extend_head(input, times=self.temporal_padding * 2) + memory = ( + input[:, :, mem_size:].detach() + if (mem_size != 0 and memory_state != MemoryState.DISABLED) + else None + ) + if ( + memory_state != MemoryState.DISABLED + and not self.training + and (self.memory_device is not None) + ): + self.memory = memory + if self.memory_device == "cpu" and self.memory is not None: + self.memory = self.memory.to("cpu") + return super().forward(input) + + def _load_from_state_dict( + self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ): + if self.inflation_mode != "none": + state_dict = modify_state_dict( + self, + state_dict, + prefix, + inflate_weight_fn=partial(inflate_weight, position="tail"), + inflate_bias_fn=partial(inflate_bias, position="tail"), + ) + super()._load_from_state_dict( + state_dict, + prefix, + local_metadata, + (strict and self.inflation_mode == "none"), + missing_keys, + unexpected_keys, + error_msgs, + ) + + +def init_causal_conv3d( + *args, + inflation_mode: _inflation_mode_t, + **kwargs, +): + """ + Initialize a Causal-3D convolution layer. + Parameters: + inflation_mode: Listed as below. It's compatible with all the 3D-VAE checkpoints we have. + - none: No inflation will be conducted. + The loading logic of state dict will fall back to default. + - tail / replicate: Refer to the definition of `InflatedCausalConv3d`. + """ + return InflatedCausalConv3d(*args, inflation_mode=inflation_mode, **kwargs) diff --git a/modules/seedvr/src/models/video_vae_v3/modules/inflated_lib.py b/modules/seedvr/src/models/video_vae_v3/modules/inflated_lib.py new file mode 100644 index 000000000..6feda1751 --- /dev/null +++ b/modules/seedvr/src/models/video_vae_v3/modules/inflated_lib.py @@ -0,0 +1,156 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from enum import Enum +from typing import Optional +import numpy as np +import torch +from diffusers.models.normalization import RMSNorm +from einops import rearrange +from torch import Tensor, nn +from ....common.logger import get_logger + + +logger = get_logger(__name__) + + +class MemoryState(Enum): + """ + State[Disabled]: No memory bank will be enabled. + State[Initializing]: The model is handling the first clip, + need to reset / initialize the memory bank. + State[Active]: There has been some data in the memory bank. + """ + + DISABLED = 0 + INITIALIZING = 1 + ACTIVE = 2 + + +def causal_norm_wrapper(norm_layer: nn.Module, x: torch.Tensor) -> torch.Tensor: + if isinstance(norm_layer, (nn.LayerNorm, RMSNorm)): + if x.ndim == 4: + x = rearrange(x, "b c h w -> b h w c") + x = norm_layer(x) + x = rearrange(x, "b h w c -> b c h w") + return x + if x.ndim == 5: + x = rearrange(x, "b c t h w -> b t h w c") + x = norm_layer(x) + x = rearrange(x, "b t h w c -> b c t h w") + return x + if isinstance(norm_layer, (nn.GroupNorm, nn.BatchNorm2d, nn.SyncBatchNorm)): + if x.ndim <= 4: + return norm_layer(x) + if x.ndim == 5: + t = x.size(2) + x = rearrange(x, "b c t h w -> (b t) c h w") + x = norm_layer(x) + x = rearrange(x, "(b t) c h w -> b c t h w", t=t) + return x + raise NotImplementedError + + +def remove_head(tensor: Tensor, times: int = 1) -> Tensor: + """ + Remove duplicated first frame features in the up-sampling process. + """ + if times == 0: + return tensor + return torch.cat(tensors=(tensor[:, :, :1], tensor[:, :, times + 1 :]), dim=2) + + +def extend_head( + tensor: Tensor, times: Optional[int] = 2, memory: Optional[Tensor] = None +) -> Tensor: + """ + When memory is None: + - Duplicate first frame features in the down-sampling process. + When memory is not None: + - Concatenate memory features with the input features to keep temporal consistency. + """ + if times == 0: + return tensor + if memory is not None: + return torch.cat((memory.to(tensor), tensor), dim=2) + else: + tile_repeat = np.ones(tensor.ndim).astype(int) + tile_repeat[2] = times + return torch.cat(tensors=(torch.tile(tensor[:, :, :1], list(tile_repeat)), tensor), dim=2) + + +def inflate_weight(weight_2d: torch.Tensor, weight_3d: torch.Tensor, inflation_mode: str): + """ + Inflate a 2D convolution weight matrix to a 3D one. + Parameters: + weight_2d: The weight matrix of 2D conv to be inflated. + weight_3d: The weight matrix of 3D conv to be initialized. + inflation_mode: the mode of inflation + """ + assert inflation_mode in ["constant", "replicate"] + assert weight_3d.shape[:2] == weight_2d.shape[:2] + with torch.no_grad(): + if inflation_mode == "replicate": + depth = weight_3d.size(2) + weight_3d.copy_(weight_2d.unsqueeze(2).repeat(1, 1, depth, 1, 1) / depth) + else: + weight_3d.fill_(0.0) + weight_3d[:, :, -1].copy_(weight_2d) + return weight_3d + + +def inflate_bias(bias_2d: torch.Tensor, bias_3d: torch.Tensor, inflation_mode: str): + """ + Inflate a 2D convolution bias tensor to a 3D one + Parameters: + bias_2d: The bias tensor of 2D conv to be inflated. + bias_3d: The bias tensor of 3D conv to be initialized. + inflation_mode: Placeholder to align `inflate_weight`. + """ + assert bias_3d.shape == bias_2d.shape + with torch.no_grad(): + bias_3d.copy_(bias_2d) + return bias_3d + + +def modify_state_dict(layer, state_dict, prefix, inflate_weight_fn, inflate_bias_fn): + """ + the main function to inflated 2D parameters to 3D. + """ + weight_name = prefix + "weight" + bias_name = prefix + "bias" + if weight_name in state_dict: + weight_2d = state_dict[weight_name] + if weight_2d.dim() == 4: + # Assuming the 2D weights are 4D tensors (out_channels, in_channels, h, w) + weight_3d = inflate_weight_fn( + weight_2d=weight_2d, + weight_3d=layer.weight, + inflation_mode=layer.inflation_mode, + ) + state_dict[weight_name] = weight_3d + else: + return state_dict + # It's a 3d state dict, should not do inflation on both bias and weight. + if bias_name in state_dict: + bias_2d = state_dict[bias_name] + if bias_2d.dim() == 1: + # Assuming the 2D biases are 1D tensors (out_channels,) + bias_3d = inflate_bias_fn( + bias_2d=bias_2d, + bias_3d=layer.bias, + inflation_mode=layer.inflation_mode, + ) + state_dict[bias_name] = bias_3d + return state_dict diff --git a/modules/seedvr/src/models/video_vae_v3/modules/types.py b/modules/seedvr/src/models/video_vae_v3/modules/types.py new file mode 100644 index 000000000..5a030d2d2 --- /dev/null +++ b/modules/seedvr/src/models/video_vae_v3/modules/types.py @@ -0,0 +1,76 @@ +# // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# // +# // Licensed under the Apache License, Version 2.0 (the "License"); +# // you may not use this file except in compliance with the License. +# // You may obtain a copy of the License at +# // +# // http://www.apache.org/licenses/LICENSE-2.0 +# // +# // Unless required by applicable law or agreed to in writing, software +# // distributed under the License is distributed on an "AS IS" BASIS, +# // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# // See the License for the specific language governing permissions and +# // limitations under the License. + +from enum import Enum +from typing import Dict, Literal, NamedTuple, Optional +import torch + +_receptive_field_t = Literal["half", "full"] +_inflation_mode_t = Literal["none", "tail", "replicate"] +_memory_device_t = Optional[Literal["cpu", "same"]] +_gradient_checkpointing_t = Optional[Literal["half", "full"]] +_selective_checkpointing_t = Optional[Literal["coarse", "fine"]] + +class DiagonalGaussianDistribution: + def __init__(self, mean: torch.Tensor, logvar: torch.Tensor): + self.mean = mean + self.logvar = torch.clamp(logvar, -30.0, 20.0) + self.std = torch.exp(0.5 * self.logvar) + self.var = torch.exp(self.logvar) + + def mode(self) -> torch.Tensor: + return self.mean + + def sample(self) -> torch.FloatTensor: + return self.mean + self.std * torch.randn_like(self.mean) + + def kl(self) -> torch.Tensor: + return 0.5 * torch.sum( + self.mean**2 + self.var - 1.0 - self.logvar, + dim=list(range(1, self.mean.ndim)), + ) + +class MemoryState(Enum): + """ + State[Disabled]: No memory bank will be enabled. + State[Initializing]: The model is handling the first clip, need to reset the memory bank. + State[Active]: There has been some data in the memory bank. + State[Unset]: Error state, indicating users didn't pass correct memory state in. + """ + + DISABLED = 0 + INITIALIZING = 1 + ACTIVE = 2 + UNSET = 3 + + +class QuantizerOutput(NamedTuple): + latent: torch.Tensor + extra_loss: torch.Tensor + statistics: Dict[str, torch.Tensor] + + +class CausalAutoencoderOutput(NamedTuple): + sample: torch.Tensor + latent: torch.Tensor + posterior: Optional[DiagonalGaussianDistribution] + + +class CausalEncoderOutput(NamedTuple): + latent: torch.Tensor + posterior: Optional[DiagonalGaussianDistribution] + + +class CausalDecoderOutput(NamedTuple): + sample: torch.Tensor diff --git a/modules/seedvr/src/models/video_vae_v3/modules/video_vae.py.old b/modules/seedvr/src/models/video_vae_v3/modules/video_vae.py.old new file mode 100644 index 000000000..2696c32ca --- /dev/null +++ b/modules/seedvr/src/models/video_vae_v3/modules/video_vae.py.old @@ -0,0 +1,936 @@ +# Copyright (c) 2023 HuggingFace Team +# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates. +# SPDX-License-Identifier: Apache License, Version 2.0 (the "License") +# +# This file has been modified by ByteDance Ltd. and/or its affiliates. on 1st June 2025 +# +# Original file was released under Apache License, Version 2.0 (the "License"), with the full license text +# available at http://www.apache.org/licenses/LICENSE-2.0. +# +# This modified file is released under the same license. + +from contextlib import nullcontext +from typing import Optional, Tuple, Literal, Callable, Union +import torch +import torch.nn as nn +from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution +from einops import rearrange +from ....common.half_precision_fixes import safe_pad_operation +from ....common.logger import get_logger +from .causal_inflation_lib import InflatedCausalConv3d, causal_norm_wrapper, init_causal_conv3d, remove_head +from .context_parallel_lib import causal_conv_gather_outputs, causal_conv_slice_inputs +from .global_config import set_norm_limit +from .types import CausalAutoencoderOutput, CausalDecoderOutput, CausalEncoderOutput, MemoryState, _inflation_mode_t, _memory_device_t, _receptive_field_t, _selective_checkpointing_t + + +logger = get_logger(__name__) # pylint: disable=invalid-name + +# Fake func, no checkpointing is required for inference +def gradient_checkpointing(module: Union[Callable, nn.Module], *args, enabled: bool, **kwargs): + return module(*args, **kwargs) + +class ResnetBlock2D(nn.Module): + r""" + A Resnet block. + + Parameters: + in_channels (`int`): The number of channels in the input. + out_channels (`int`, *optional*, default to be `None`): + The number of output channels for the first conv2d layer. + If None, same as `in_channels`. + dropout (`float`, *optional*, defaults to `0.0`): The dropout probability to use. + """ + + def __init__( + self, *, in_channels: int, out_channels: Optional[int] = None, dropout: float = 0.0 + ): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + + self.nonlinearity = nn.SiLU() + + self.norm1 = torch.nn.GroupNorm( + num_groups=32, num_channels=in_channels, eps=1e-6, affine=True + ) + + self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + + self.norm2 = torch.nn.GroupNorm( + num_groups=32, num_channels=out_channels, eps=1e-6, affine=True + ) + + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) + + self.use_in_shortcut = self.in_channels != out_channels + + self.conv_shortcut = None + if self.use_in_shortcut: + self.conv_shortcut = nn.Conv2d( + in_channels, out_channels, kernel_size=1, stride=1, padding=0 + ) + + def forward(self, input_tensor: torch.Tensor) -> torch.Tensor: + hidden = input_tensor + + hidden = self.norm1(hidden) + hidden = self.nonlinearity(hidden) + hidden = self.conv1(hidden) + + hidden = self.norm2(hidden) + hidden = self.nonlinearity(hidden) + hidden = self.dropout(hidden) + hidden = self.conv2(hidden) + + if self.conv_shortcut is not None: + input_tensor = self.conv_shortcut(input_tensor) + + output_tensor = input_tensor + hidden + + return output_tensor + +class Upsample3D(nn.Module): + """A 3D upsampling layer.""" + + def __init__( + self, + channels: int, + inflation_mode: _inflation_mode_t = "tail", + temporal_up: bool = False, + spatial_up: bool = True, + slicing: bool = False, + ): + super().__init__() + self.channels = channels + self.conv = init_causal_conv3d( + self.channels, self.channels, kernel_size=3, padding=1, inflation_mode=inflation_mode + ) + + self.temporal_up = temporal_up + self.spatial_up = spatial_up + self.temporal_ratio = 2 if temporal_up else 1 + self.spatial_ratio = 2 if spatial_up else 1 + self.slicing = slicing + + upscale_ratio = (self.spatial_ratio**2) * self.temporal_ratio + self.upscale_conv = nn.Conv3d( + self.channels, self.channels * upscale_ratio, kernel_size=1, padding=0 + ) + identity = ( + torch.eye(self.channels).repeat(upscale_ratio, 1).reshape_as(self.upscale_conv.weight) + ) + + self.upscale_conv.weight.data.copy_(identity) + nn.init.zeros_(self.upscale_conv.bias) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + memory_state: MemoryState, + ) -> torch.FloatTensor: + return gradient_checkpointing( + self.custom_forward, + hidden_states, + memory_state, + enabled=self.training and self.gradient_checkpointing, + ) + + def custom_forward( + self, + hidden_states: torch.FloatTensor, + memory_state: MemoryState, + ) -> torch.FloatTensor: + assert hidden_states.shape[1] == self.channels + + if self.slicing: + split_size = hidden_states.size(2) // 2 + hidden_states = list( + hidden_states.split([split_size, hidden_states.size(2) - split_size], dim=2) + ) + else: + hidden_states = [hidden_states] + + for i in range(len(hidden_states)): + hidden_states[i] = self.upscale_conv(hidden_states[i]) + hidden_states[i] = rearrange( + hidden_states[i], + "b (x y z c) f h w -> b c (f z) (h x) (w y)", + x=self.spatial_ratio, + y=self.spatial_ratio, + z=self.temporal_ratio, + ) + + # [Overridden] For causal temporal conv + if self.temporal_up and memory_state != MemoryState.ACTIVE: + hidden_states[0] = remove_head(hidden_states[0]) + + if self.slicing: + hidden_states = self.conv(hidden_states, memory_state=memory_state) + return torch.cat(hidden_states, dim=2) + else: + return self.conv(hidden_states[0], memory_state=memory_state) + + +class Downsample3D(nn.Module): + """A 3D downsampling layer.""" + + def __init__( + self, + channels: int, + inflation_mode: _inflation_mode_t = "tail", + temporal_down: bool = False, + spatial_down: bool = True, + ): + super().__init__() + self.channels = channels + self.temporal_down = temporal_down + self.spatial_down = spatial_down + + self.temporal_ratio = 2 if temporal_down else 1 + self.spatial_ratio = 2 if spatial_down else 1 + + self.temporal_kernel = 3 if temporal_down else 1 + self.spatial_kernel = 3 if spatial_down else 1 + + self.conv = init_causal_conv3d( + self.channels, + self.channels, + kernel_size=(self.temporal_kernel, self.spatial_kernel, self.spatial_kernel), + stride=(self.temporal_ratio, self.spatial_ratio, self.spatial_ratio), + padding=((1 if self.temporal_down else 0), 0, 0), + inflation_mode=inflation_mode, + ) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + memory_state: MemoryState, + ) -> torch.FloatTensor: + return gradient_checkpointing( + self.custom_forward, + hidden_states, + memory_state, + enabled=self.training and self.gradient_checkpointing, + ) + + def custom_forward( + self, + hidden_states: torch.FloatTensor, + memory_state: MemoryState, + ) -> torch.FloatTensor: + + assert hidden_states.shape[1] == self.channels + + if self.spatial_down: + hidden_states = safe_pad_operation(hidden_states, (0, 1, 0, 1), mode="constant", value=0) + + hidden_states = self.conv(hidden_states, memory_state=memory_state) + return hidden_states + + +class ResnetBlock3D(ResnetBlock2D): + def __init__( + self, + *args, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + **kwargs, + ): + super().__init__(*args, **kwargs) + self.conv1 = init_causal_conv3d( + self.in_channels, + self.out_channels, + kernel_size=3, + stride=1, + padding=1, + inflation_mode=inflation_mode, + ) + + self.conv2 = init_causal_conv3d( + self.out_channels, + self.out_channels, + kernel_size=(1, 3, 3) if time_receptive_field == "half" else (3, 3, 3), + stride=1, + padding=(0, 1, 1) if time_receptive_field == "half" else (1, 1, 1), + inflation_mode=inflation_mode, + ) + + if self.use_in_shortcut: + self.conv_shortcut = init_causal_conv3d( + self.in_channels, + self.out_channels, + kernel_size=1, + stride=1, + padding=0, + bias=(self.conv_shortcut.bias is not None), + inflation_mode=inflation_mode, + ) + self.gradient_checkpointing = False + + def forward(self, input_tensor: torch.Tensor, memory_state: MemoryState = MemoryState.UNSET): + return gradient_checkpointing( + self.custom_forward, + input_tensor, + memory_state, + enabled=self.training and self.gradient_checkpointing, + ) + + def custom_forward( + self, input_tensor: torch.Tensor, memory_state: MemoryState = MemoryState.UNSET + ): + assert memory_state != MemoryState.UNSET + hidden_states = input_tensor + + hidden_states = causal_norm_wrapper(self.norm1, hidden_states) + hidden_states = self.nonlinearity(hidden_states) + hidden_states = self.conv1(hidden_states, memory_state=memory_state) + + hidden_states = causal_norm_wrapper(self.norm2, hidden_states) + hidden_states = self.nonlinearity(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states, memory_state=memory_state) + + if self.conv_shortcut is not None: + input_tensor = self.conv_shortcut(input_tensor, memory_state=memory_state) + + output_tensor = input_tensor + hidden_states + + return output_tensor + + +class DownEncoderBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + add_downsample: bool = True, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + temporal_down: bool = True, + spatial_down: bool = True, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock3D( + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + self.downsamplers = None + if add_downsample: + # Todo: Refactor this line before V5 Image VAE Training. + self.downsamplers = nn.ModuleList( + [ + Downsample3D( + channels=out_channels, + inflation_mode=inflation_mode, + temporal_down=temporal_down, + spatial_down=spatial_down, + ) + ] + ) + + def forward( + self, hidden_states: torch.FloatTensor, memory_state: MemoryState + ) -> torch.FloatTensor: + for resnet in self.resnets: + hidden_states = resnet(hidden_states, memory_state=memory_state) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, memory_state=memory_state) + + return hidden_states + + +class UpDecoderBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + add_upsample: bool = True, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + temporal_up: bool = True, + spatial_up: bool = True, + slicing: bool = False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + resnets.append( + ResnetBlock3D( + in_channels=input_channels, + out_channels=out_channels, + dropout=dropout, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + self.upsamplers = None + # Todo: Refactor this line before V5 Image VAE Training. + if add_upsample: + self.upsamplers = nn.ModuleList( + [ + Upsample3D( + channels=out_channels, + inflation_mode=inflation_mode, + temporal_up=temporal_up, + spatial_up=spatial_up, + slicing=slicing, + ) + ] + ) + + def forward( + self, hidden_states: torch.FloatTensor, memory_state: MemoryState + ) -> torch.FloatTensor: + for resnet in self.resnets: + hidden_states = resnet(hidden_states, memory_state=memory_state) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, memory_state=memory_state) + + return hidden_states + + +class UNetMidBlock3D(nn.Module): + def __init__( + self, + channels: int, + dropout: float = 0.0, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + ): + super().__init__() + self.resnets = nn.ModuleList( + [ + ResnetBlock3D( + in_channels=channels, + out_channels=channels, + dropout=dropout, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ), + ResnetBlock3D( + in_channels=channels, + out_channels=channels, + dropout=dropout, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ), + ] + ) + + def forward(self, hidden_states: torch.Tensor, memory_state: MemoryState): + for resnet in self.resnets: + hidden_states = resnet(hidden_states, memory_state) + return hidden_states + + +class Encoder3D(nn.Module): + r""" + The `Encoder` layer of a variational autoencoder that encodes + its input into a latent representation. + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 2, + double_z: bool = True, + temporal_down_num: int = 2, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + selective_checkpointing: Tuple[_selective_checkpointing_t] = ("none",), + ): + super().__init__() + self.layers_per_block = layers_per_block + + self.temporal_down_num = temporal_down_num + + self.conv_in = init_causal_conv3d( + in_channels, + block_out_channels[0], + kernel_size=3, + stride=1, + padding=1, + inflation_mode=inflation_mode, + ) + + self.down_blocks = nn.ModuleList([]) + + # down + output_channel = block_out_channels[0] + for i in range(len(block_out_channels)): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + is_temporal_down_block = i >= len(block_out_channels) - self.temporal_down_num - 1 + # Note: take the last one + + down_block = DownEncoderBlock3D( + num_layers=self.layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + add_downsample=not is_final_block, + temporal_down=is_temporal_down_block, + spatial_down=True, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + self.down_blocks.append(down_block) + + # mid + self.mid_block = UNetMidBlock3D( + channels=block_out_channels[-1], + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + + # out + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[-1], num_groups=32, eps=1e-6 + ) + self.conv_act = nn.SiLU() + + conv_out_channels = 2 * out_channels if double_z else out_channels + self.conv_out = init_causal_conv3d( + block_out_channels[-1], conv_out_channels, 3, padding=1, inflation_mode=inflation_mode + ) + + assert len(selective_checkpointing) == len(self.down_blocks) + self.set_gradient_checkpointing(selective_checkpointing) + + def set_gradient_checkpointing(self, checkpointing_types): + gradient_checkpointing = [] + for down_block, sac_type in zip(self.down_blocks, checkpointing_types): + if sac_type == "coarse": + gradient_checkpointing.append(True) + elif sac_type == "fine": + for n, m in down_block.named_modules(): + if hasattr(m, "gradient_checkpointing"): + m.gradient_checkpointing = True + logger.debug(f"set gradient_checkpointing: {n}") + gradient_checkpointing.append(False) + else: + gradient_checkpointing.append(False) + self.gradient_checkpointing = gradient_checkpointing + logger.info(f"[Encoder3D] gradient_checkpointing: {checkpointing_types}") + + def forward(self, sample: torch.FloatTensor, memory_state: MemoryState) -> torch.FloatTensor: + r"""The forward method of the `Encoder` class.""" + sample = self.conv_in(sample, memory_state=memory_state) + # down + for down_block, sac in zip(self.down_blocks, self.gradient_checkpointing): + sample = gradient_checkpointing( + down_block, + sample, + memory_state=memory_state, + enabled=self.training and sac, + ) + + # middle + sample = self.mid_block(sample, memory_state=memory_state) + + # post-process + sample = causal_norm_wrapper(self.conv_norm_out, sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample, memory_state=memory_state) + + return sample + + +class Decoder3D(nn.Module): + r""" + The `Decoder` layer of a variational autoencoder that + decodes its latent representation into an output sample. + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 2, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + temporal_up_num: int = 2, + slicing_up_num: int = 0, + selective_checkpointing: Tuple[_selective_checkpointing_t] = ("none",), + ): + super().__init__() + self.layers_per_block = layers_per_block + self.temporal_up_num = temporal_up_num + + self.conv_in = init_causal_conv3d( + in_channels, + block_out_channels[-1], + kernel_size=3, + stride=1, + padding=1, + inflation_mode=inflation_mode, + ) + + self.up_blocks = nn.ModuleList([]) + + # mid + self.mid_block = UNetMidBlock3D( + channels=block_out_channels[-1], + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i in range(len(reversed_block_out_channels)): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + + is_final_block = i == len(block_out_channels) - 1 + is_temporal_up_block = i < self.temporal_up_num + is_slicing_up_block = i >= len(block_out_channels) - slicing_up_num + # Note: Keep symmetric + + up_block = UpDecoderBlock3D( + num_layers=self.layers_per_block + 1, + in_channels=prev_output_channel, + out_channels=output_channel, + add_upsample=not is_final_block, + temporal_up=is_temporal_up_block, + slicing=is_slicing_up_block, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + self.up_blocks.append(up_block) + + # out + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[0], num_groups=32, eps=1e-6 + ) + self.conv_act = nn.SiLU() + self.conv_out = init_causal_conv3d( + block_out_channels[0], out_channels, 3, padding=1, inflation_mode=inflation_mode + ) + + assert len(selective_checkpointing) == len(self.up_blocks) + self.set_gradient_checkpointing(selective_checkpointing) + + def set_gradient_checkpointing(self, checkpointing_types): + gradient_checkpointing = [] + for up_block, sac_type in zip(self.up_blocks, checkpointing_types): + if sac_type == "coarse": + gradient_checkpointing.append(True) + elif sac_type == "fine": + for n, m in up_block.named_modules(): + if hasattr(m, "gradient_checkpointing"): + m.gradient_checkpointing = True + logger.debug(f"set gradient_checkpointing: {n}") + gradient_checkpointing.append(False) + else: + gradient_checkpointing.append(False) + self.gradient_checkpointing = gradient_checkpointing + logger.info(f"[Decoder3D] gradient_checkpointing: {checkpointing_types}") + + def forward(self, sample: torch.FloatTensor, memory_state: MemoryState) -> torch.FloatTensor: + r"""The forward method of the `Decoder` class.""" + + sample = self.conv_in(sample, memory_state=memory_state) + + # middle + sample = self.mid_block(sample, memory_state=memory_state) + + # up + for up_block, sac in zip(self.up_blocks, self.gradient_checkpointing): + sample = gradient_checkpointing( + up_block, + sample, + memory_state=memory_state, + enabled=self.training and sac, + ) + + # post-process + sample = causal_norm_wrapper(self.conv_norm_out, sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample, memory_state=memory_state) + + return sample + + +class VideoAutoencoderKL(nn.Module): + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + block_out_channels: Tuple[int] = (64,), + layers_per_block: int = 1, + latent_channels: int = 4, + use_quant_conv: bool = True, + use_post_quant_conv: bool = True, + enc_selective_checkpointing: Tuple[_selective_checkpointing_t] = ("none",), + dec_selective_checkpointing: Tuple[_selective_checkpointing_t] = ("none",), + temporal_scale_num: int = 0, + slicing_up_num: int = 0, + inflation_mode: _inflation_mode_t = "tail", + time_receptive_field: _receptive_field_t = "half", + slicing_sample_min_size: int = None, + spatial_downsample_factor: int = 16, + temporal_downsample_factor: int = 8, + freeze_encoder: bool = False, + ): + super().__init__() + self.spatial_downsample_factor = spatial_downsample_factor + self.temporal_downsample_factor = temporal_downsample_factor + self.freeze_encoder = freeze_encoder + if slicing_sample_min_size is None: + slicing_sample_min_size = temporal_downsample_factor + self.slicing_sample_min_size = slicing_sample_min_size + self.slicing_latent_min_size = slicing_sample_min_size // (2**temporal_scale_num) + + # pass init params to Encoder + self.encoder = Encoder3D( + in_channels=in_channels, + out_channels=latent_channels, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + double_z=True, + temporal_down_num=temporal_scale_num, + selective_checkpointing=enc_selective_checkpointing, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + + # pass init params to Decoder + self.decoder = Decoder3D( + in_channels=latent_channels, + out_channels=out_channels, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + # [Override] add temporal_up_num parameter + temporal_up_num=temporal_scale_num, + slicing_up_num=slicing_up_num, + selective_checkpointing=dec_selective_checkpointing, + inflation_mode=inflation_mode, + time_receptive_field=time_receptive_field, + ) + + self.quant_conv = ( + init_causal_conv3d( + in_channels=2 * latent_channels, + out_channels=2 * latent_channels, + kernel_size=1, + inflation_mode=inflation_mode, + ) + if use_quant_conv + else None + ) + self.post_quant_conv = ( + init_causal_conv3d( + in_channels=latent_channels, + out_channels=latent_channels, + kernel_size=1, + inflation_mode=inflation_mode, + ) + if use_post_quant_conv + else None + ) + + self.use_slicing = False + + def enable_slicing(self): + self.use_slicing = True + + def disable_slicing(self): + self.use_slicing = False + + def encode(self, x: torch.FloatTensor) -> CausalEncoderOutput: + if x.ndim == 4: + x = x.unsqueeze(2) + h = self.slicing_encode(x) + p = DiagonalGaussianDistribution(h) + z = p.sample() + return CausalEncoderOutput(z, p) + + def decode(self, z: torch.FloatTensor) -> CausalDecoderOutput: + if z.ndim == 4: + z = z.unsqueeze(2) + x = self.slicing_decode(z) + return CausalDecoderOutput(x) + + def _encode(self, x: torch.Tensor, memory_state: MemoryState) -> torch.Tensor: + x = causal_conv_slice_inputs(x, self.slicing_sample_min_size, memory_state=memory_state) + h = self.encoder(x, memory_state=memory_state) + h = self.quant_conv(h, memory_state=memory_state) if self.quant_conv is not None else h + h = causal_conv_gather_outputs(h) + return h + + def _decode(self, z: torch.Tensor, memory_state: MemoryState) -> torch.Tensor: + z = causal_conv_slice_inputs(z, self.slicing_latent_min_size, memory_state=memory_state) + z = ( + self.post_quant_conv(z, memory_state=memory_state) + if self.post_quant_conv is not None + else z + ) + x = self.decoder(z, memory_state=memory_state) + x = causal_conv_gather_outputs(x) + return x + + def slicing_encode(self, x: torch.Tensor) -> torch.Tensor: + sp_size = 1 + if self.use_slicing and (x.shape[2] - 1) > self.slicing_sample_min_size * sp_size: + x_slices = x[:, :, 1:].split(split_size=self.slicing_sample_min_size * sp_size, dim=2) + encoded_slices = [ + self._encode( + torch.cat((x[:, :, :1], x_slices[0]), dim=2), + memory_state=MemoryState.INITIALIZING, + ) + ] + for x_idx in range(1, len(x_slices)): + encoded_slices.append( + self._encode(x_slices[x_idx], memory_state=MemoryState.ACTIVE) + ) + return torch.cat(encoded_slices, dim=2) + else: + return self._encode(x, memory_state=MemoryState.DISABLED) + + def slicing_decode(self, z: torch.Tensor) -> torch.Tensor: + sp_size = 1 + if self.use_slicing and (z.shape[2] - 1) > self.slicing_latent_min_size * sp_size: + z_slices = z[:, :, 1:].split(split_size=self.slicing_latent_min_size * sp_size, dim=2) + decoded_slices = [ + self._decode( + torch.cat((z[:, :, :1], z_slices[0]), dim=2), + memory_state=MemoryState.INITIALIZING, + ) + ] + for z_idx in range(1, len(z_slices)): + decoded_slices.append( + self._decode(z_slices[z_idx], memory_state=MemoryState.ACTIVE) + ) + return torch.cat(decoded_slices, dim=2) + else: + return self._decode(z, memory_state=MemoryState.DISABLED) + + def forward(self, x: torch.FloatTensor) -> CausalAutoencoderOutput: + with torch.no_grad() if self.freeze_encoder else nullcontext(): + z, p = self.encode(x) + x = self.decode(z).sample + return CausalAutoencoderOutput(x, z, p) + + def preprocess(self, x: torch.Tensor): + # x should in [B, C, T, H, W], [B, C, H, W] + assert x.ndim == 4 or x.size(2) % self.temporal_downsample_factor == 1 + return x + + def postprocess(self, x: torch.Tensor): + # x should in [B, C, T, H, W], [B, C, H, W] + return x + + def set_causal_slicing( + self, + *, + split_size: Optional[int], + memory_device: _memory_device_t, + ): + assert ( + split_size is None or memory_device is not None + ), "if split_size is set, memory_device must not be None." + if split_size is not None: + self.enable_slicing() + self.slicing_sample_min_size = split_size + self.slicing_latent_min_size = split_size // self.temporal_downsample_factor + else: + self.disable_slicing() + for module in self.modules(): + if isinstance(module, InflatedCausalConv3d): + module.set_memory_device(memory_device) + + def set_memory_limit(self, conv_max_mem: Optional[float], norm_max_mem: Optional[float]): + set_norm_limit(norm_max_mem) + for m in self.modules(): + if isinstance(m, InflatedCausalConv3d): + m.set_memory_limit(conv_max_mem if conv_max_mem is not None else float("inf")) + + +class VideoAutoencoderKLWrapper(VideoAutoencoderKL): + def __init__( + self, *args, spatial_downsample_factor: int, temporal_downsample_factor: int, **kwargs + ): + self.spatial_downsample_factor = spatial_downsample_factor + self.temporal_downsample_factor = temporal_downsample_factor + super().__init__(*args, **kwargs) + + def forward(self, x) -> CausalAutoencoderOutput: + z, _, p = self.encode(x) + x, _ = self.decode(z) + return CausalAutoencoderOutput(x, z, None, p) + + def encode(self, x) -> CausalEncoderOutput: + if x.ndim == 4: + x = x.unsqueeze(2) + p = super().encode(x).latent_dist + z = p.sample().squeeze(2) + return CausalEncoderOutput(z, None, p) + + def decode(self, z) -> CausalDecoderOutput: + if z.ndim == 4: + z = z.unsqueeze(2) + x = super().decode(z).sample.squeeze(2) + return CausalDecoderOutput(x, None) + + def preprocess(self, x): + # x should in [B, C, T, H, W], [B, C, H, W] + assert x.ndim == 4 or x.size(2) % 4 == 1 + return x + + def postprocess(self, x): + # x should in [B, C, T, H, W], [B, C, H, W] + return x + + def set_causal_slicing( + self, + *, + split_size: Optional[int], + memory_device: Optional[Literal["cpu", "same"]], + ): + assert ( + split_size is None or memory_device is not None + ), "if split_size is set, memory_device must not be None." + if split_size is not None: + self.enable_slicing() + else: + self.disable_slicing() + self.slicing_sample_min_size = split_size + if split_size is not None: + self.slicing_latent_min_size = split_size // self.temporal_downsample_factor + for module in self.modules(): + if isinstance(module, InflatedCausalConv3d): + module.set_memory_device(memory_device) diff --git a/modules/seedvr/src/models/video_vae_v3/s8_c16_t4_inflation_sd3.yaml b/modules/seedvr/src/models/video_vae_v3/s8_c16_t4_inflation_sd3.yaml new file mode 100644 index 000000000..949122667 --- /dev/null +++ b/modules/seedvr/src/models/video_vae_v3/s8_c16_t4_inflation_sd3.yaml @@ -0,0 +1,28 @@ +act_fn: silu +block_out_channels: + - 128 + - 256 + - 512 + - 512 +down_block_types: + - DownEncoderBlock3D + - DownEncoderBlock3D + - DownEncoderBlock3D + - DownEncoderBlock3D +in_channels: 3 +latent_channels: 16 +layers_per_block: 2 +norm_num_groups: 32 +out_channels: 3 +slicing_sample_min_size: 4 +temporal_scale_num: 2 +inflation_mode: pad +up_block_types: + - UpDecoderBlock3D + - UpDecoderBlock3D + - UpDecoderBlock3D + - UpDecoderBlock3D +spatial_downsample_factor: 8 +temporal_downsample_factor: 4 +use_quant_conv: False +use_post_quant_conv: False diff --git a/modules/seedvr/src/optimization/__init__.py b/modules/seedvr/src/optimization/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/modules/seedvr/src/optimization/memory_manager.py b/modules/seedvr/src/optimization/memory_manager.py new file mode 100644 index 000000000..39feada97 --- /dev/null +++ b/modules/seedvr/src/optimization/memory_manager.py @@ -0,0 +1,111 @@ +""" +Memory management module for SeedVR2 +Handles VRAM usage, cache management, and memory optimization + +Extracted from: seedvr2.py (lines 373-405, 607-626, 1016-1044) +""" + +import torch +from ..common.cache import Cache +from ..models.dit_v2.rope import RotaryEmbeddingBase + + +def preinitialize_rope_cache(runner) -> None: + """ + 🚀 Pre-initialize RoPE cache to avoid OOM at first launch + + Args: + runner: The model runner containing DiT and VAE models + """ + + # Create dummy tensors to simulate common shapes + # Format: [batch, channels, frames, height, width] for vid_shape + # Format: [batch, seq_len] for txt_shape + common_shapes = [ + # Common video resolutions + (torch.tensor([[1, 3, 3]], dtype=torch.long), torch.tensor([[77]], dtype=torch.long)), # 1 frame, 77 tokens + (torch.tensor([[4, 3, 3]], dtype=torch.long), torch.tensor([[77]], dtype=torch.long)), # 4 frames + (torch.tensor([[5, 3, 3]], dtype=torch.long), torch.tensor([[77]], dtype=torch.long)), # 5 frames (4n+1 format) + (torch.tensor([[1, 4, 4]], dtype=torch.long), torch.tensor([[77]], dtype=torch.long)), # Higher resolution + ] + + # Create mock cache for pre-initialization + + temp_cache = Cache() + + # Access RoPE modules in DiT (recursive search) + def find_rope_modules(module): + rope_modules = [] + for name, child in module.named_modules(): + if hasattr(child, 'get_freqs') and callable(child.get_freqs): + rope_modules.append((name, child)) + return rope_modules + + rope_modules = find_rope_modules(runner.dit) + + # Pre-calculate for each RoPE module found + for _name, rope_module in rope_modules: + # Temporarily move module to CPU if necessary + original_device = next(rope_module.parameters()).device if list(rope_module.parameters()) else torch.device('cpu') + rope_module.to('cpu') + + for vid_shape, txt_shape in common_shapes: + cache_key = f"720pswin_by_size_bysize_{tuple(vid_shape[0].tolist())}_sd3.mmrope_freqs_3d" + + def compute_freqs(): + # Calculate with reduced dimensions to avoid OOM + with torch.no_grad(): + # Detect RoPE module type + module_type = type(rope_module).__name__ + + if module_type == 'NaRotaryEmbedding3d': + # NaRotaryEmbedding3d: only takes shape (vid_shape) + return rope_module.get_freqs(vid_shape.cpu()) + else: + # Standard RoPE: takes vid_shape and txt_shape + return rope_module.get_freqs(vid_shape.cpu(), txt_shape.cpu()) + + # Store in cache + temp_cache(cache_key, compute_freqs) + + rope_module.to(original_device) + + # Copy temporary cache to runner cache + if hasattr(runner, 'cache'): + runner.cache.cache.update(temp_cache.cache) + else: + runner.cache = temp_cache + + +def clear_rope_cache(runner) -> None: + """ + 🧹 Clear RoPE cache to free VRAM + + Args: + runner: The model runner containing the cache + """ + if hasattr(runner, 'cache') and hasattr(runner.cache, 'cache'): + # Count entries before cleanup + cache_size = len(runner.cache.cache) + + # Free all tensors from cache + for key, value in runner.cache.cache.items(): + if isinstance(value, (tuple, list)): + for item in value: + if hasattr(item, 'cpu'): + item.cpu() + del item + elif hasattr(value, 'cpu'): + value.cpu() + del value + + # Clear the cache + runner.cache.cache.clear() + + if hasattr(runner, 'dit'): + cleared_lru_count = 0 + for module in runner.dit.modules(): + if isinstance(module, RotaryEmbeddingBase): + if hasattr(module.get_axial_freqs, 'cache_clear'): + module.get_axial_freqs.cache_clear() + cleared_lru_count += 1 diff --git a/modules/seedvr/src/optimization/performance.py b/modules/seedvr/src/optimization/performance.py new file mode 100644 index 000000000..83cba0462 --- /dev/null +++ b/modules/seedvr/src/optimization/performance.py @@ -0,0 +1,157 @@ +""" +Performance optimization module for SeedVR2 +Contains optimized tensor operations and video processing functions + +Extracted from: seedvr2.py (lines 1633-1730) +""" + +import torch +from typing import List, Union + + +def optimized_video_rearrange(video_tensors: List[torch.Tensor]) -> List[torch.Tensor]: + """ + 🚀 OPTIMIZED version of video rearrangement + Replaces slow loops with vectorized operations + + Transforms: + - 3D: c h w -> t c h w (with t=1) + - 4D: c t h w -> t c h w + + Expected gains: 5-10x faster than naive loops + + Args: + video_tensors: List of video tensors to rearrange + + Returns: + List of rearranged tensors in t c h w format + """ + if not video_tensors: + return [] + + # 🔍 Analyze dimensions to optimize processing + videos_3d = [] + videos_4d = [] + indices_3d = [] + indices_4d = [] + + for i, video in enumerate(video_tensors): + if video.ndim == 3: + videos_3d.append(video) + indices_3d.append(i) + else: # ndim == 4 + videos_4d.append(video) + indices_4d.append(i) + + # 🎯 Prepare final result + samples = [None] * len(video_tensors) + + # 🚀 BATCH PROCESSING for 3D videos (c h w -> 1 c h w) + if videos_3d: + # Method 1: Stack + permute (faster than rearrange) + # c h w -> c 1 h w -> 1 c h w + batch_3d = torch.stack([v.unsqueeze(1) for v in videos_3d]) # [batch, c, 1, h, w] + batch_3d = batch_3d.permute(0, 2, 1, 3, 4) # [batch, 1, c, h, w] + + for i, idx in enumerate(indices_3d): + samples[idx] = batch_3d[i] # [1, c, h, w] + + # 🚀 BATCH PROCESSING for 4D videos (c t h w -> t c h w) + if videos_4d: + # Check if all 4D videos have the same shape for maximum optimization + shapes = [v.shape for v in videos_4d] + if len(set(shapes)) == 1: + # 🎯 MAXIMUM OPTIMIZATION: All shapes identical + # Stack + permute in single operation + batch_4d = torch.stack(videos_4d) # [batch, c, t, h, w] + batch_4d = batch_4d.permute(0, 2, 1, 3, 4) # [batch, t, c, h, w] + + for i, idx in enumerate(indices_4d): + samples[idx] = batch_4d[i] # [t, c, h, w] + else: + # 🔄 FALLBACK: Different shapes, optimized individual processing + for i, idx in enumerate(indices_4d): + # Use permute instead of rearrange (faster) + samples[idx] = videos_4d[i].permute(1, 0, 2, 3) # c t h w -> t c h w + + return samples + + +def optimized_single_video_rearrange(video: torch.Tensor) -> torch.Tensor: + """ + 🚀 OPTIMIZED version for single video tensor + Replaces rearrange() with native PyTorch operations + + Transforms: + - 3D: c h w -> 1 c h w (add temporal dimension) + - 4D: c t h w -> t c h w (permute dimensions) + + Expected gains: 2-5x faster than rearrange() + + Args: + video: Input video tensor + + Returns: + Rearranged tensor with temporal dimension first + """ + if video.ndim == 3: + # c h w -> 1 c h w (add temporal dimension t=1) + return video.unsqueeze(0) + else: # ndim == 4 + # c t h w -> t c h w (permute channels and temporal) + return video.permute(1, 0, 2, 3) + + +def optimized_sample_to_image_format(sample: torch.Tensor) -> torch.Tensor: + """ + 🚀 OPTIMIZED version to convert sample to image format + Replaces rearrange() with native PyTorch operations + + Transforms: + - 3D: c h w -> 1 h w c (add temporal dimension + permute to image format) + - 4D: t c h w -> t h w c (permute to image format) + + Expected gains: 2-5x faster than rearrange() + + Args: + sample: Input sample tensor + + Returns: + Tensor in image format (channels last) + """ + if sample.ndim == 3: + # c h w -> 1 h w c (add temporal dimension then permute) + return sample.unsqueeze(0).permute(0, 2, 3, 1) + else: # ndim == 4 + # t c h w -> t h w c (permute channels to last) + return sample.permute(0, 2, 3, 1) + + +def temporal_latent_blending(latents1: torch.Tensor, latents2: torch.Tensor, blend_frames: int) -> torch.Tensor: + """ + 🎨 Temporal blending in latent space to avoid discontinuities + + Args: + latents1: Latents from previous batch (end frames) + latents2: Latents from current batch (start frames) + blend_frames: Number of frames to blend + + Returns: + Blended latents for smooth transition + """ + if latents1.shape[0] != latents2.shape[0]: + # Adjust dimensions if necessary + min_frames = min(latents1.shape[0], latents2.shape[0]) + latents1 = latents1[:min_frames] + latents2 = latents2[:min_frames] + + # Create linear blending weights + # Frame 0: 100% latents1, 0% latents2 + # Frame n: 0% latents1, 100% latents2 + weights1 = torch.linspace(1.0, 0.0, blend_frames).view(-1, 1, 1, 1).to(latents1.device) + weights2 = torch.linspace(0.0, 1.0, blend_frames).view(-1, 1, 1, 1).to(latents2.device) + + # Apply blending + blended_latents = weights1 * latents1 + weights2 * latents2 + + return blended_latents diff --git a/modules/seedvr/src/utils/__init__.py b/modules/seedvr/src/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/modules/seedvr/src/utils/color_fix.py b/modules/seedvr/src/utils/color_fix.py new file mode 100644 index 000000000..a8b0da509 --- /dev/null +++ b/modules/seedvr/src/utils/color_fix.py @@ -0,0 +1,138 @@ +import torch +from PIL import Image +from torch import Tensor +from torch.nn import functional as F +from ..common.half_precision_fixes import safe_pad_operation, safe_interpolate_operation +from torchvision.transforms import ToTensor, ToPILImage + +def adain_color_fix(target: Image, source: Image): + # Convert images to tensors + to_tensor = ToTensor() + target_tensor = to_tensor(target).unsqueeze(0) + source_tensor = to_tensor(source).unsqueeze(0) + + # Apply adaptive instance normalization + result_tensor = adaptive_instance_normalization(target_tensor, source_tensor) + + # Convert tensor back to image + to_image = ToPILImage() + result_image = to_image(result_tensor.squeeze(0).clamp_(0.0, 1.0)) + + return result_image + +def wavelet_color_fix(target: Image, source: Image): + # Convert images to tensors + to_tensor = ToTensor() + target_tensor = to_tensor(target).unsqueeze(0) + source_tensor = to_tensor(source).unsqueeze(0) + + # Apply wavelet reconstruction + result_tensor = wavelet_reconstruction(target_tensor, source_tensor) + + # Convert tensor back to image + to_image = ToPILImage() + result_image = to_image(result_tensor.squeeze(0).clamp_(0.0, 1.0)) + + return result_image + +def calc_mean_std(feat: Tensor, eps=1e-5): + """Calculate mean and std for adaptive_instance_normalization. + Args: + feat (Tensor): 4D tensor. + eps (float): A small value added to the variance to avoid + divide-by-zero. Default: 1e-5. + """ + size = feat.size() + assert len(size) == 4, 'The input feature should be 4D tensor.' + b, c = size[:2] + feat_var = feat.view(b, c, -1).var(dim=2) + eps + feat_std = feat_var.sqrt().view(b, c, 1, 1) + feat_mean = feat.view(b, c, -1).mean(dim=2).view(b, c, 1, 1) + return feat_mean, feat_std + +def adaptive_instance_normalization(content_feat:Tensor, style_feat:Tensor): + """Adaptive instance normalization. + Adjust the reference features to have the similar color and illuminations + as those in the degradate features. + Args: + content_feat (Tensor): The reference feature. + style_feat (Tensor): The degradate features. + """ + size = content_feat.size() + style_mean, style_std = calc_mean_std(style_feat) + content_mean, content_std = calc_mean_std(content_feat) + normalized_feat = (content_feat - content_mean.expand(size)) / content_std.expand(size) + return normalized_feat * style_std.expand(size) + style_mean.expand(size) + +def wavelet_blur(image: Tensor, radius: int): + """ + Apply wavelet blur to the input tensor. + """ + # input shape: (1, 3, H, W) + # convolution kernel + kernel_vals = [ + [0.0625, 0.125, 0.0625], + [0.125, 0.25, 0.125], + [0.0625, 0.125, 0.0625], + ] + kernel = torch.tensor(kernel_vals, dtype=image.dtype, device=image.device) + # add channel dimensions to the kernel to make it a 4D tensor + kernel = kernel[None, None] + # repeat the kernel across all input channels + kernel = kernel.repeat(3, 1, 1, 1) + image = safe_pad_operation(image, (radius, radius, radius, radius), mode='replicate') + # apply convolution + output = F.conv2d(image, kernel, groups=3, dilation=radius) + return output + +def wavelet_decomposition(image: Tensor, levels=5): + """ + Apply wavelet decomposition to the input tensor. + This function only returns the low frequency & the high frequency. + """ + high_freq = torch.zeros_like(image) + for i in range(levels): + radius = 2 ** i + low_freq = wavelet_blur(image, radius) + high_freq += (image - low_freq) + image = low_freq + + return high_freq, low_freq + + + +def wavelet_reconstruction(content_feat:Tensor, style_feat:Tensor): + """ + Apply wavelet decomposition, so that the content will have the same color as the style. + """ + # Vérifier et ajuster les dimensions si nécessaire + if content_feat.shape != style_feat.shape: + # Redimensionner style_feat pour correspondre à content_feat + target_shape = content_feat.shape + if len(target_shape) >= 3: # Au moins 3 dimensions + # Utiliser interpolation pour ajuster les dimensions spatiales + style_feat = safe_interpolate_operation( + style_feat, + size=target_shape[-2:], # Dernières 2 dimensions (H, W) + mode='bilinear', + align_corners=False + ) + + # calculate the wavelet decomposition of the content feature + content_high_freq, content_low_freq = wavelet_decomposition(content_feat) + del content_low_freq + # calculate the wavelet decomposition of the style feature + style_high_freq, style_low_freq = wavelet_decomposition(style_feat) + del style_high_freq + + # Vérification finale avant addition + if content_high_freq.shape != style_low_freq.shape: + style_low_freq = safe_interpolate_operation( + style_low_freq, + size=content_high_freq.shape[-2:], + mode='bilinear', + align_corners=False + ) + + # reconstruct the content feature with the style's high frequency + return content_high_freq + style_low_freq diff --git a/modules/seedvr/test.py b/modules/seedvr/test.py new file mode 100644 index 000000000..b8c53f874 --- /dev/null +++ b/modules/seedvr/test.py @@ -0,0 +1,56 @@ +import os +import numpy as np +import torch +from PIL import Image +from torchvision.transforms import ToPILImage +from .src.core.generation import generation_loop +from .src.core.model_manager import configure_runner + + + +device = 'cuda' +dtype = torch.bfloat16 +model_dir = 'seedvr2_models' +model = 'seedvr2_ema_3b_fp16.safetensors' +resolution = 1024 +seed = 100 +cfg = 1.0 +input_image = '/home/vlado/generative/Samples/cutie-512.png' + +to_pil = ToPILImage() +runner = None +loaded_model = None + + +def upscale_image(model_name:str, image_path:str): + global runner, loaded_model + if (runner is None) or (loaded_model != model_name): + runner = configure_runner(model_name, model_dir, device=device, dtype=dtype) + loaded_model = model_name + + image = Image.open(image_path).convert("RGB") + image_tensor = np.array(image) + image_tensor = torch.from_numpy(image_tensor).to(device=device, dtype=dtype).unsqueeze(0) / 255.0 + + result_tensor = generation_loop( + runner=runner, + images=image_tensor, + cfg_scale=cfg, + seed=seed, + res_w=resolution, + batch_size=1, + temporal_overlap=0, + device=device, + ) + image = to_pil(result_tensor.squeeze().permute((2, 0, 1))) + + output_path = os.path.join('/tmp', os.path.basename(image_path)) + + image.save(output_path, quality=95) + return image + + +if __name__ == "__main__": + output_image = upscale_image(model, input_image) + print('input:', input_image) + print('output:', output_image) diff --git a/modules/shared.py b/modules/shared.py index a969c81b0..fcaa05f1b 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -70,9 +70,8 @@ state = shared_state.State() # early select backend backend = Backend.DIFFUSERS -if not hasattr(cmd_opts, "use_openvino"): - cmd_opts.use_openvino = False if cmd_opts.use_openvino: # override for openvino + os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX') from modules.intel.openvino import get_device_list as get_openvino_device_list # pylint: disable=ungrouped-imports elif cmd_opts.use_ipex or devices.has_xpu(): from modules.intel.ipex import ipex_init @@ -84,6 +83,11 @@ elif cmd_opts.use_directml: ok, e = directml_init() if not ok: log.error(f'DirectML initialization failed: {e}') +elif cmd_opts.use_rocm or devices.has_rocm(): + from modules.rocm import rocm_init + ok, e = rocm_init() + if not ok: + log.error(f'ROCm initialization failed: {e}') devices.backend = devices.get_backend(cmd_opts) devices.device = devices.get_optimal_device() mem_stat = memory_stats() @@ -155,6 +159,8 @@ options_templates.update(options_section(('sd', "Model Loading"), { })) options_templates.update(options_section(('model_options', "Model Options"), { + "model_modular_sep": OptionInfo("

Modular Pipelines

", "", gr.HTML), + "model_modular_enable": OptionInfo(False, "Enable modular pipelines (experimental)"), "model_sd3_sep": OptionInfo("

Stable Diffusion 3.x

", "", gr.HTML), "model_sd3_disable_te5": OptionInfo(False, "Disable T5 text encoder"), "model_h1_sep": OptionInfo("

HiDream

", "", gr.HTML), @@ -170,9 +176,11 @@ options_templates.update(options_section(('offload', "Model Offloading"), { "diffusers_offload_nonblocking": OptionInfo(False, "Non-blocking move operations"), "offload_balanced_sep": OptionInfo("

Balanced Offload

", "", gr.HTML), "diffusers_offload_pre": OptionInfo(True, "Offload during pre-forward"), + "diffusers_offload_streams": OptionInfo(False, "Offload using streams"), "diffusers_offload_min_gpu_memory": OptionInfo(startup_offload_min_gpu, "Offload low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), "diffusers_offload_max_gpu_memory": OptionInfo(startup_offload_max_gpu, "Offload GPU high watermark", gr.Slider, {"minimum": 0.1, "maximum": 1, "step": 0.01 }), "diffusers_offload_max_cpu_memory": OptionInfo(0.90, "Offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False }), + "models_not_to_offload": OptionInfo("", "Model types not to offload"), "diffusers_offload_always": OptionInfo(startup_offload_always, "Modules to always offload"), "diffusers_offload_never": OptionInfo(startup_offload_never, "Modules to never offload"), "offload_group_sep": OptionInfo("

Group Offload

", "", gr.HTML), @@ -183,7 +191,7 @@ options_templates.update(options_section(('offload', "Model Offloading"), { })) options_templates.update(options_section(("quantization", "Model Quantization"), { - "models_not_to_quant": OptionInfo("", "List of model types not to quantize"), + "models_not_to_quant": OptionInfo("", "Model types not to quantize"), "sdnq_quantize_sep": OptionInfo("

SDNQ: SD.Next Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "TE", "LLM", "Control", "VAE"]}), @@ -193,6 +201,9 @@ options_templates.update(options_section(("quantization", "Model Quantization"), "sdnq_modules_to_not_convert": OptionInfo("", "Modules to not convert"), "sdnq_modules_dtype_dict": OptionInfo("{}", "Modules dtype dict"), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1}), + "sdnq_svd_rank": OptionInfo(32, "SVD rank size", gr.Slider, {"minimum": 1, "maximum": 512, "step": 1}), + "sdnq_svd_steps": OptionInfo(8, "SVD steps", gr.Slider, {"minimum": 1, "maximum": 128, "step": 1}), + "sdnq_use_svd": OptionInfo(False, "Use SVD quantization", gr.Checkbox), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize convolutional layers", gr.Checkbox), "sdnq_dequantize_compile": OptionInfo(devices.has_triton(), "Dequantize using torch.compile", gr.Checkbox), "sdnq_use_quantized_matmul": OptionInfo(False, "Use quantized MatMul", gr.Checkbox), @@ -248,8 +259,8 @@ options_templates.update(options_section(('vae_encoder', "Variational Auto Encod "sd_vae": OptionInfo("Automatic", "VAE model", gr.Dropdown, lambda: {"choices": shared_items.sd_vae_items()}, refresh=shared_items.refresh_vae_list), "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, {"choices": ['default', 'true', 'false']}), "no_half_vae": OptionInfo(False if not cmd_opts.use_openvino else True, "Full precision (--no-half-vae)"), - "diffusers_vae_slicing": OptionInfo(True, "VAE slicing", gr.Checkbox), - "diffusers_vae_tiling": OptionInfo(cmd_opts.lowvram or cmd_opts.medvram, "VAE tiling", gr.Checkbox), + "diffusers_vae_slicing": OptionInfo(cmd_opts.lowvram or cmd_opts.medvram, "VAE slicing", gr.Checkbox), + "diffusers_vae_tiling": OptionInfo(cmd_opts.lowvram, "VAE tiling", gr.Checkbox), "diffusers_vae_tile_size": OptionInfo(0, "VAE tile size", gr.Slider, {"minimum": 0, "maximum": 4096, "step": 8 }), "diffusers_vae_tile_overlap": OptionInfo(0.25, "VAE tile overlap", gr.Slider, {"minimum": 0, "maximum": 0.95, "step": 0.05 }), "remote_vae_type": OptionInfo('raw', "Remote VAE image type", gr.Dropdown, {"choices": ['raw', 'jpg', 'png']}), @@ -282,6 +293,8 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "cross_attention_sep": OptionInfo("

Cross Attention

", "", gr.HTML), "cross_attention_optimization": OptionInfo(startup_cross_attention, "Attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention()}), + "attention_": OptionInfo("

Cross Attention

", "", gr.HTML), + "attention_slicing": OptionInfo('Default', "Attention slicing", gr.Radio, {"choices": ['Default', 'Enabled', 'Disabled']}), "sdp_options": OptionInfo(startup_sdp_options, "SDP options", gr.CheckboxGroup, {"choices": startup_sdp_choices}), "xformers_options": OptionInfo(['Flash attention'], "xFormers options", gr.CheckboxGroup, {"choices": ['Flash attention'] }), "dynamic_attention_slice_rate": OptionInfo(0.5, "Dynamic Attention slicing rate in GB", gr.Slider, {"minimum": 0.01, "maximum": max(gpu_memory,4), "step": 0.01}), @@ -354,6 +367,14 @@ options_templates.update(options_section(('advanced', "Pipeline Modifiers"), { "pab_spacial_skip_start": OptionInfo(100, "PAB spacial skip start", gr.Slider, {"minimum": 0, "maximum": 1000, "step": 1}), "pab_spacial_skip_end": OptionInfo(800, "PAB spacial skip end", gr.Slider, {"minimum": 0, "maximum": 1000, "step": 1}), + "cache_dit_sep": OptionInfo("

Cache-DiT

", "", gr.HTML), + "cache_dit_enabled": OptionInfo(False, "Cache-DiT enabled"), + "cache_dit_calibrator": OptionInfo("None", "Cache-DiT calibrator", gr.Radio, {"choices": ["None", "TaylorSeer", "FoCa"]}), + "cache_dit_fcompute": OptionInfo(-1, "Cache-DiT F-compute blocks", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), + "cache_dit_bcompute": OptionInfo(-1, "Cache-DiT B-compute blocks", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), + "cache_dit_threshold": OptionInfo(-1, "Cache-DiT residual diff threshold", gr.Slider, {"minimum": -1.0, "maximum": 1.0, "step": 0.01}), + "cache_dit_warmup": OptionInfo(-1, "Cache-DiT warmup steps", gr.Slider, {"minimum": -1, "maximum": 50, "step": 1}), + "faster_cache__sep": OptionInfo("

Faster Cache

", "", gr.HTML), "faster_cache_enabled": OptionInfo(False, "FasterCache cache enabled"), "fc_spacial_skip_range": OptionInfo(2, "FasterCache spacial skip range", gr.Slider, {"minimum": 1, "maximum": 4, "step": 1}), @@ -453,7 +474,7 @@ options_templates.update(options_section(('system-paths', "System Paths"), { options_templates.update(options_section(('saving-images', "Image Options"), { "samples_save": OptionInfo(True, "Save all generated images"), - "keep_incomplete": OptionInfo(True, "Keep incomplete images"), + "keep_incomplete": OptionInfo(True, "Save interrupted images"), "samples_format": OptionInfo('jpg', 'File format', gr.Dropdown, {"choices": ["jpg", "png", "webp", "tiff", "jp2", "jxl"]}), "jpeg_quality": OptionInfo(90, "Image quality", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}), "img_max_size_mp": OptionInfo(1000, "Maximum image size (MP)", gr.Slider, {"minimum": 100, "maximum": 2000, "step": 1}), @@ -600,7 +621,6 @@ options_templates.update(options_section(('postprocessing', "Postprocessing"), { "detailer_iou": OptionInfo(0.5, "Max overlap", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.05, "visible": False}), "detailer_sigma_adjust": OptionInfo(1.0, "Detailer sigma adjust", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.05, "visible": False}), "detailer_sigma_adjust_max": OptionInfo(1.0, "Detailer sigma end", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.05, "visible": False}), - # "detailer_resolution": OptionInfo(1024, "Detailer resolution", gr.Slider, {"minimum": 256, "maximum": 4096, "step": 8, "visible": False}), "detailer_min_size": OptionInfo(0.0, "Min object size", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.05, "visible": False}), "detailer_max_size": OptionInfo(1.0, "Max object size", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.05, "visible": False}), "detailer_padding": OptionInfo(20, "Item padding", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1, "visible": False}), @@ -611,6 +631,9 @@ options_templates.update(options_section(('postprocessing', "Postprocessing"), { "detailer_unload": OptionInfo(False, "Move detailer model to CPU when complete"), "detailer_augment": OptionInfo(True, "Detailer use model augment"), + "postprocessing_sep_seedvt": OptionInfo("

SeedVT

", "", gr.HTML), + "seedvt_cfg_scale": OptionInfo(3.5, "SeedVR CFG Scale", gr.Slider, {"minimum": 1, "maximum": 15, "step": 1}), + "postprocessing_sep_face_restore": OptionInfo("

Face Restore

", "", gr.HTML), "face_restoration_model": OptionInfo("None", "Face restoration", gr.Radio, lambda: {"choices": ['None'] + [x.name() for x in face_restorers]}), "code_former_weight": OptionInfo(0.2, "CodeFormer weight parameter", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}), @@ -665,6 +688,7 @@ options_templates.update(options_section(('huggingface', "Huggingface"), { "diffuser_cache_config": OptionInfo(True, "Use cached model config when available"), "huggingface_token": OptionInfo('', 'HuggingFace token', gr.Textbox, {"lines": 2}), "hf_transfer_mode": OptionInfo("rust", "HuggingFace download method", gr.Radio, {"choices": ['requests', 'rust', 'xet']}), + "huggingface_mirror": OptionInfo('', 'HuggingFace mirror', gr.Textbox), "diffusers_model_load_variant": OptionInfo("default", "Preferred Model variant", gr.Radio, {"choices": ['default', 'fp32', 'fp16']}), "diffusers_vae_load_variant": OptionInfo("default", "Preferred VAE variant", gr.Radio, {"choices": ['default', 'fp32', 'fp16']}), diff --git a/modules/shared_items.py b/modules/shared_items.py index 8eac5e070..f09df5769 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -56,6 +56,7 @@ pipelines = { 'Bria': getattr(diffusers, 'DiffusionPipeline', None), 'hdm': getattr(diffusers, 'DiffusionPipeline', None), 'HunyuanImage': getattr(diffusers, 'DiffusionPipeline', None), + 'X-Omni': getattr(diffusers, 'DiffusionPipeline', None), } @@ -120,7 +121,6 @@ def list_crossattention(): "Scaled-Dot-Product", "xFormers", "Batch matrix-matrix", - "Split attention", "Dynamic Attention BMM" ] diff --git a/modules/styles.py b/modules/styles.py index ec3ca71e9..4ea4509a6 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -123,7 +123,7 @@ def apply_wildcards_to_prompt(prompt, all_wildcards, seed=-1, silent=False): if replaced and not silent: shared.log.debug(f'Apply wildcards: {replaced} path="{shared.opts.wildcards_dir}" type=style time={t1-t0:.2f}') if (len(replaced_file) > 0 or len(not_found) > 0) and not silent: - shared.log.debug(f'Apply wildcards: found={replaced_file} missing={not_found} path="{shared.opts.wildcards_dir}" type=file time={t2-t2:.2f} ') + shared.log.debug(f'Apply wildcards: found={replaced_file} missing={not_found} path="{shared.opts.wildcards_dir}" type=file seed={seed} time={t2-t2:.2f}') if old_state is not None: random.setstate(old_state) return prompt @@ -168,6 +168,7 @@ def apply_styles_to_extra(p, style: Style): params = [] settings = [] skipped = [] + for k, v in extra.items(): k = k.lower().replace(' ', '_') if k in name_map: # rename some fields @@ -322,16 +323,24 @@ class StyleDatabase: jobid = shared.state.begin('Styles') parsed_positive = [] parsed_negative = [] + random_state = random.getstate() + for i in range(len(prompts)): + if seeds[i]> 0: + random.seed(seeds[i]) prompt = prompts[i] prompt = apply_styles_to_prompt(prompt, [self.find_style(x).prompt for x in styles]) prompt = apply_wildcards_to_prompt(prompt, [self.find_style(x).wildcards for x in styles], seeds[i]) parsed_positive.append(prompt) for i in range(len(negatives)): + if seeds[i]> 0: + random.seed(seeds[i]) prompt = negatives[i] prompt = apply_styles_to_prompt(prompt, [self.find_style(x).negative_prompt for x in styles]) prompt = apply_wildcards_to_prompt(prompt, [self.find_style(x).wildcards for x in styles], seeds[i]) parsed_negative.append(prompt) + + random.setstate(random_state) shared.state.end(jobid) return parsed_positive, parsed_negative diff --git a/modules/taesd/taehv.py b/modules/taesd/taehv.py index 0f1828e45..a8ba17469 100644 --- a/modules/taesd/taehv.py +++ b/modules/taesd/taehv.py @@ -62,6 +62,8 @@ def apply_model_with_memblocks(model, x, parallel, show_progress_bar): if x.ndim == 4: x = x.unsqueeze(0) assert x.ndim == 5, f"TAEHV operates on NTCHW tensors, but got {x.ndim}-dim tensor" + if x.shape[1] == 16 and x.shape[2] != 16: + x = x.transpose(1,2) # NCTHW to NTCHW N, T, C, H, W = x.shape if parallel: x = x.reshape(N*T, C, H, W) @@ -159,6 +161,7 @@ class TAEHV(nn.Module): decoder_space_upscale: whether spatial upsampling is enabled for each block. upsampling can be disabled for a cheaper preview. """ super().__init__() + from modules import shared self.encoder = nn.Sequential( conv(TAEHV.image_channels, 64), nn.ReLU(inplace=True), TPool(64, 2), conv(64, 64, stride=2, bias=False), MemBlock(64, 64), MemBlock(64, 64), MemBlock(64, 64), @@ -168,13 +171,32 @@ class TAEHV(nn.Module): ) n_f = [256, 128, 64, 64] self.frames_to_trim = 2**sum(decoder_time_upscale) - 1 - self.decoder = nn.Sequential( - Clamp(), conv(TAEHV.latent_channels, n_f[0]), nn.ReLU(inplace=True), - MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), nn.Upsample(scale_factor=2 if decoder_space_upscale[0] else 1), TGrow(n_f[0], 1), conv(n_f[0], n_f[1], bias=False), - MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), nn.Upsample(scale_factor=2 if decoder_space_upscale[1] else 1), TGrow(n_f[1], 2 if decoder_time_upscale[0] else 1), conv(n_f[1], n_f[2], bias=False), - MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), nn.Upsample(scale_factor=2 if decoder_space_upscale[2] else 1), TGrow(n_f[2], 2 if decoder_time_upscale[1] else 1), conv(n_f[2], n_f[3], bias=False), - nn.ReLU(inplace=True), conv(n_f[3], TAEHV.image_channels), - ) + + if shared.opts.taesd_layers == 1: + self.decoder = nn.Sequential( + Clamp(), conv(TAEHV.latent_channels, n_f[0]), nn.ReLU(inplace=True), + MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), nn.Upsample(scale_factor=2 if decoder_space_upscale[0] else 1), TGrow(n_f[0], 1), conv(n_f[0], n_f[1], bias=False), + MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), nn.Identity(), TGrow(n_f[1], 2 if decoder_time_upscale[0] else 1), conv(n_f[1], n_f[2], bias=False), + MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), nn.Identity(), TGrow(n_f[2], 2 if decoder_time_upscale[1] else 1), conv(n_f[2], n_f[3], bias=False), + nn.ReLU(inplace=True), conv(n_f[3], TAEHV.image_channels), + ) + elif shared.opts.taesd_layers == 2: + self.decoder = nn.Sequential( + Clamp(), conv(TAEHV.latent_channels, n_f[0]), nn.ReLU(inplace=True), + MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), nn.Upsample(scale_factor=2 if decoder_space_upscale[0] else 1), TGrow(n_f[0], 1), conv(n_f[0], n_f[1], bias=False), + MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), nn.Upsample(scale_factor=2 if decoder_space_upscale[1] else 1), TGrow(n_f[1], 2 if decoder_time_upscale[0] else 1), conv(n_f[1], n_f[2], bias=False), + MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), nn.Identity(), TGrow(n_f[2], 2 if decoder_time_upscale[1] else 1), conv(n_f[2], n_f[3], bias=False), + nn.ReLU(inplace=True), conv(n_f[3], TAEHV.image_channels), + ) + else: + self.decoder = nn.Sequential( + Clamp(), conv(TAEHV.latent_channels, n_f[0]), nn.ReLU(inplace=True), + MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), nn.Upsample(scale_factor=2 if decoder_space_upscale[0] else 1), TGrow(n_f[0], 1), conv(n_f[0], n_f[1], bias=False), + MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), nn.Upsample(scale_factor=2 if decoder_space_upscale[1] else 1), TGrow(n_f[1], 2 if decoder_time_upscale[0] else 1), conv(n_f[1], n_f[2], bias=False), + MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), nn.Upsample(scale_factor=2 if decoder_space_upscale[2] else 1), TGrow(n_f[2], 2 if decoder_time_upscale[1] else 1), conv(n_f[2], n_f[3], bias=False), + nn.ReLU(inplace=True), conv(n_f[3], TAEHV.image_channels), + ) + if checkpoint_path is not None: self.load_state_dict(self.patch_tgrow_layers(torch.load(checkpoint_path, map_location="cpu", weights_only=True))) diff --git a/modules/txt2img.py b/modules/txt2img.py index 3397d4fe9..08d4f1bf2 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -14,6 +14,7 @@ def txt2img(id_task, state, vae_type, tiling, hidiffusion, detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, n_iter, batch_size, + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, clip_skip, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, @@ -55,6 +56,11 @@ def txt2img(id_task, state, batch_size=batch_size, n_iter=n_iter, steps=steps, + guidance_name=guidance_name, + guidance_scale=guidance_scale, + guidance_rescale=guidance_rescale, + guidance_start=guidance_start, + guidance_stop=guidance_stop, cfg_scale=cfg_scale, image_cfg_scale=image_cfg_scale, diffusers_guidance_rescale=diffusers_guidance_rescale, diff --git a/modules/ui.py b/modules/ui.py index d87f02c70..f9daddb35 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -28,6 +28,7 @@ plaintext_to_html = ui_common.plaintext_to_html # compatibility item infotext_to_html = ui_common.infotext_to_html # compatibility item create_sampler_and_steps_selection = ui_sections.create_sampler_and_steps_selection # compatibility item ui_system_tabs = None # required for system-info +interfaces = [] if not shared.cmd_opts.share and not shared.cmd_opts.listen: @@ -132,7 +133,8 @@ def create_ui(startup_timer = None): ui_extensions.create_ui() timer.startup.record("ui-extensions") - interfaces = [] + global interfaces # pylint: disable=global-statement + interfaces.clear() interfaces += [(txt2img_interface, "Text", "txt2img")] interfaces += [(img2img_interface, "Image", "img2img")] if control_interface is not None: diff --git a/modules/ui_caption.py b/modules/ui_caption.py index 85d3ca0bc..e0ef85933 100644 --- a/modules/ui_caption.py +++ b/modules/ui_caption.py @@ -28,6 +28,7 @@ def update_clip_params(*args): def create_ui(): + shared.log.debug('UI initialize: tab=caption') with gr.Row(equal_height=False, variant='compact', elem_classes="caption", elem_id="caption_tab"): with gr.Column(variant='compact', elem_id='interrogate_input'): with gr.Row(): @@ -47,7 +48,7 @@ def create_ui(): with gr.Row(): vlm_max_tokens = gr.Slider(label='VLM max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1, elem_id='vlm_max_tokens') vlm_num_beams = gr.Slider(label='VLM num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1, elem_id='vlm_num_beams') - vlm_temperature = gr.Slider(label='VLM Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01, elem_id='vlm_temperature') + vlm_temperature = gr.Slider(label='VLM Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.0, maximum=1.0, step=0.01, elem_id='vlm_temperature') with gr.Row(): vlm_top_k = gr.Slider(label='Top-K', value=shared.opts.interrogate_vlm_top_k, minimum=0, maximum=99, step=1, elem_id='vlm_top_k') vlm_top_p = gr.Slider(label='Top-P', value=shared.opts.interrogate_vlm_top_p, minimum=0.0, maximum=1.0, step=0.01, elem_id='vlm_top_p') diff --git a/modules/ui_common.py b/modules/ui_common.py index 1c5499cf7..7fa684190 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -5,7 +5,7 @@ import shutil import platform import subprocess import gradio as gr -from modules import call_queue, shared, errors, ui_sections, ui_symbols, ui_components, generation_parameters_copypaste, images, scripts_manager, script_callbacks, infotext +from modules import call_queue, shared, errors, ui_sections, ui_symbols, ui_components, generation_parameters_copypaste, images, scripts_manager, script_callbacks, infotext, processing folder_symbol = ui_symbols.folder @@ -19,8 +19,12 @@ def gr_show(visible=True): def update_generation_info(generation_info, html_info, img_index): try: + if len(generation_info) == 0 and processing.processed is not None: + generation_info = processing.processed.js() or {} + if len(generation_info) == 0: + return html_info, html_info generation_json = json.loads(generation_info) - if len(generation_json["infotexts"]) == 0: + if len(generation_json.get("infotexts", [])) == 0: return html_info, 'no infotexts found' if img_index == -1: img_index = 0 @@ -29,8 +33,8 @@ def update_generation_info(generation_info, html_info, img_index): info = generation_json["infotexts"][img_index] html_info_formatted = infotext_to_html(info) return html_info, html_info_formatted - except Exception: - pass + except Exception as e: + shared.log.trace(f'Update info: info="{generation_info}" {e}') return html_info, html_info @@ -238,7 +242,7 @@ def open_folder(result_gallery, gallery_index = 0): subprocess.Popen(["xdg-open", path]) # pylint: disable=consider-using-with -def create_output_panel(tabname, preview=True, prompt=None, height=None, transfer=True, scale=1): +def create_output_panel(tabname, preview=True, prompt=None, height=None, transfer=True, scale=1, result_info=None): with gr.Column(variant='panel', elem_id=f"{tabname}_results", scale=scale): with gr.Group(elem_id=f"{tabname}_gallery_container"): if tabname == "txt2img": @@ -286,10 +290,11 @@ def create_output_panel(tabname, preview=True, prompt=None, height=None, transfe generation_info = gr.Textbox(visible=False, elem_id=f'generation_info_{tabname}') generation_info_button = gr.Button(visible=False, elem_id=f"{tabname}_generation_info_button") + result_field = result_info or html_info_formatted generation_info_button.click(fn=update_generation_info, show_progress=False, _js="(x, y, z) => [x, y, selected_gallery_index()]", # triggered on gallery change from js inputs=[generation_info, html_info, html_info], - outputs=[html_info, html_info_formatted], + outputs=[html_info, result_field], ) save.click(fn=call_queue.wrap_gradio_call(save_files), show_progress=False, _js="(x, y, z, i) => [x, y, z, selected_gallery_index()]", @@ -354,11 +359,12 @@ def create_override_inputs(tab): # pylint: disable=unused-argument def reuse_seed(seed_component: gr.Number, reuse_button: gr.Button, subseed:bool=False): def reuse_click(selected_gallery_index): selected_gallery_index = int(selected_gallery_index) - from modules import processing if processing.processed is None: seed = -1 - elif selected_gallery_index >= len(processing.processed.all_seeds): - selected_gallery_index -= len(processing.processed.images) - len(processing.processed.all_seeds) # if we have more images than seeds it is likely the grid image + elif len(processing.processed.images) > len(processing.processed.all_seeds): # if we have more images than seeds it is likely the grid image + selected_gallery_index -= (len(processing.processed.images) - len(processing.processed.all_seeds)) + seed = processing.processed.all_seeds[selected_gallery_index] if not subseed else processing.processed.all_subseeds[selected_gallery_index] + elif selected_gallery_index <= len(processing.processed.all_seeds): seed = processing.processed.all_seeds[selected_gallery_index] if not subseed else processing.processed.all_subseeds[selected_gallery_index] elif len(processing.processed.all_seeds) > 0: seed = processing.processed.all_seeds[0] if not subseed else processing.processed.all_subseeds[0] diff --git a/modules/ui_control.py b/modules/ui_control.py index b75842aee..af5de9882 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -1,15 +1,9 @@ import os import time import gradio as gr -import matplotlib.pyplot as plt from modules.control import unit -from modules.control import processors # patrickvonplaten controlnet_aux -from modules.control.units import controlnet # lllyasviel ControlNet -from modules.control.units import xs # vislearn ControlNet-XS -from modules.control.units import lite # vislearn ControlNet-XS -from modules.control.units import t2iadapter # TencentARC T2I-Adapter -from modules.control.units import reference # reference pipeline -from modules import errors, shared, progress, ui_components, ui_symbols, ui_common, ui_sections, generation_parameters_copypaste, call_queue, scripts_manager, masking, images, processing_vae, timer # pylint: disable=ungrouped-imports +from modules import errors, shared, progress, generation_parameters_copypaste, call_queue, scripts_manager, masking, images, processing_vae, timer # pylint: disable=ungrouped-imports +from modules import ui_common, ui_sections, ui_guidance from modules import ui_control_helpers as helpers @@ -163,7 +157,8 @@ def create_ui(_blocks: gr.Blocks=None): mask_controls = masking.create_segment_ui() - vae_type, tiling, hidiffusion, cfg_scale, clip_skip, image_cfg_scale, guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_sections.create_advanced_inputs('control') + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_guidance.create_guidance_inputs('control') + vae_type, tiling, hidiffusion, clip_skip = ui_sections.create_advanced_inputs('control') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('control') with gr.Accordion(open=False, label="Video", elem_id="control_video", elem_classes=["small-accordion"]): @@ -188,14 +183,14 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Column(scale=9, elem_id='control-input-column', visible=True) as column_input: gr.HTML('Input

') with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-input'): + input_mode = gr.Label(value='select', visible=False) with gr.Tab('Image', id='in-image') as tab_image: - input_mode = gr.Label(value='select', visible=False) - input_image = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="editor", height=gr_height, visible=True, image_mode='RGB', elem_id='control_input_select', elem_classes=['control-image']) - input_resize = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="select", height=gr_height, visible=False, image_mode='RGB', elem_id='control_input_resize', elem_classes=['control-image']) - input_inpaint = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="sketch", height=gr_height, visible=False, image_mode='RGB', elem_id='control_input_inpaint', brush_radius=32, mask_opacity=0.6, elem_classes=['control-image']) + input_image = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="editor", height=gr_height, image_mode='RGB', elem_id='control_input_select', elem_classes=['control-image']) btn_interrogate = ui_sections.create_interrogate_button('control', what='input') - with gr.Row(): - input_buttons = [gr.Button('Select', visible=True, interactive=False), gr.Button('Inpaint', visible=True, interactive=True), gr.Button('Outpaint', visible=True, interactive=True)] + with gr.Tab('Inpaint', id='in-inpaint') as _tab_inpaint: + input_inpaint = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="sketch", height=gr_height, image_mode='RGB', elem_id='control_input_inpaint', brush_radius=32, mask_opacity=0.6, elem_classes=['control-image']) + with gr.Tab('Outpaint', id='in-outpaint') as _tab_outpaint: + input_resize = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="select", height=gr_height, image_mode='RGB', elem_id='control_input_resize', elem_classes=['control-image']) with gr.Tab('Video', id='in-video') as tab_video: input_video = gr.Video(label="Input", show_label=False, interactive=True, height=gr_height, elem_classes=['control-image']) with gr.Tab('Batch', id='in-batch') as tab_batch: @@ -217,7 +212,7 @@ def create_ui(_blocks: gr.Blocks=None): gr.HTML('Output

') with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-output') as output_tabs: with gr.Tab('Gallery', id='out-gallery'): - output_gallery, _output_gen_info, _output_html_info, _output_html_info_formatted, output_html_log = ui_common.create_output_panel("control", preview=False, prompt=prompt, height=gr_height) + output_gallery, _output_gen_info, _output_html_info, _output_html_info_formatted, output_html_log = ui_common.create_output_panel("control", preview=False, prompt=prompt, height=gr_height, result_info=result_txt) with gr.Tab('Image', id='out-image'): output_image = gr.Image(label="Output", show_label=False, type="pil", interactive=False, tool="editor", height=gr_height, elem_id='control_output_image', elem_classes=['control-image']) with gr.Tab('Video', id='out-video'): @@ -228,320 +223,17 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Tab('Preview', id='preview-image') as _tab_preview: preview_process = gr.Image(label="Preview", show_label=False, type="pil", interactive=False, height=gr_height, visible=True, elem_id='control_preview', elem_classes=['control-image']) - with gr.Accordion('Control elements', open=False, elem_id="control_elements"): - with gr.Tabs(elem_id='control-tabs') as _tabs_control_type: - with gr.Tab('ControlNet') as _tab_controlnet: - gr.HTML('ControlNet') - with gr.Row(): - extra_controls = [ - gr.Checkbox(label="Guess mode", value=False, scale=3), - ] - num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) - controlnet_ui_units = [] # list of hidable accordions - for i in range(max_units): - enabled = True if i==0 else False - with gr.Accordion(f'ControlNet unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: - with gr.Row(): - enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') - process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') - model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') - ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, f'controlnet_models_{i}_refresh') - control_mode = gr.Dropdown(label="CN Mode", choices=['default'], value='default', visible=False, elem_id=f'control_unit-{i}-mode') - model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=2.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') - control_start = gr.Slider(label="CN Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') - control_end = gr.Slider(label="CN End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') - control_tile = gr.Dropdown(label="CN Tiles", choices=[x.strip() for x in shared.opts.control_tiles.split(',') if 'x' in x], value='1x1', visible=False, elem_id=f'control_unit-{i}-tile') - reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'controlnet_unit-{i}-reset') - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'controlnet_unit-{i}-upload') - image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'controlnet_unit-{i}-reuse') - btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'controlnet_unit-{i}-preview') - image_preview = gr.Image(label="Input", type="pil", height=128, width=128, visible=False, interactive=True, show_label=False, show_download_button=False, container=False, elem_id=f'controlnet_unit-{i}-override') - controlnet_ui_units.append(unit_ui) - units.append(unit.Unit( - unit_type = 'controlnet', - index = i, - enabled = enabled, - result_txt = result_txt, - enabled_cb = enabled_cb, - reset_btn = reset_btn, - process_id = process_id, - model_id = model_id, - model_strength = model_strength, - preview_process = preview_process, - preview_btn = btn_preview, - image_upload = image_upload, - image_reuse = image_reuse, - image_preview = image_preview, - control_start = control_start, - control_end = control_end, - control_mode = control_mode, - control_tile = control_tile, - extra_controls = extra_controls, - ) - ) - if i == 0: - units[-1].enabled = True # enable first unit in group - num_controlnet_units.change(fn=helpers.display_units, inputs=[num_controlnet_units], outputs=controlnet_ui_units) - - with gr.Tab('T2I Adapter') as _tab_t2iadapter: - gr.HTML('T2I-Adapter') - with gr.Row(): - extra_controls = [ - gr.Slider(label="Control factor", minimum=0.0, maximum=1.0, step=0.05, value=1.0, scale=3), - ] - num_adapter_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) - adapter_ui_units = [] # list of hidable accordions - for i in range(max_units): - enabled = True if i==0 else False - with gr.Accordion(f'T2I-Adapter unit {i+1}', visible= i < num_adapter_units.value, elem_classes='control-unit') as unit_ui: - with gr.Row(): - enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') - process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') - model_id = gr.Dropdown(label="Adapter", choices=t2iadapter.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') - ui_common.create_refresh_button(model_id, t2iadapter.list_models, lambda: {"choices": t2iadapter.list_models(refresh=True)}, f'adapter_models_{i}_refresh') - model_strength = gr.Slider(label="T2I Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') - reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'adapter_unit-{i}-reset') - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'adapter_unit-{i}-upload') - image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'adapter_unit-{i}-reuse') - btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'adapter_unit-{i}-preview') - image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'adapter_unit-{i}-override') - adapter_ui_units.append(unit_ui) - units.append(unit.Unit( - unit_type = 't2i adapter', - index = i, - enabled = enabled, - result_txt = result_txt, - enabled_cb = enabled_cb, - reset_btn = reset_btn, - process_id = process_id, - model_id = model_id, - model_strength = model_strength, - preview_process = preview_process, - preview_btn = btn_preview, - image_upload = image_upload, - image_reuse = image_reuse, - image_preview = image_preview, - extra_controls = extra_controls, - ) - ) - if i == 0: - units[-1].enabled = True # enable first unit in group - num_adapter_units.change(fn=helpers.display_units, inputs=[num_adapter_units], outputs=adapter_ui_units) - - with gr.Tab('XS') as _tab_controlnetxs: - gr.HTML('ControlNet XS') - with gr.Row(): - extra_controls = [ - gr.Slider(label="Time embedding mix", minimum=0.0, maximum=1.0, step=0.05, value=0.0, scale=3) - ] - num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) - controlnetxs_ui_units = [] # list of hidable accordions - for i in range(max_units): - enabled = True if i==0 else False - with gr.Accordion(f'ControlNet-XS unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: - with gr.Row(): - enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') - process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') - model_id = gr.Dropdown(label="ControlNet-XS", choices=xs.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') - ui_common.create_refresh_button(model_id, xs.list_models, lambda: {"choices": xs.list_models(refresh=True)}, f'xs_models_{i}_refresh') - model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') - control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') - control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') - reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'controlnetxs_unit-{i}-reset') - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'controlnetxs_unit-{i}-upload') - image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'controlnetxs_unit-{i}-reuse') - btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'controlnetxs_unit-{i}-preview') - image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'controlnetxs_unit-{i}-override') - controlnetxs_ui_units.append(unit_ui) - units.append(unit.Unit( - unit_type = 'xs', - index = i, - enabled = enabled, - result_txt = result_txt, - enabled_cb = enabled_cb, - reset_btn = reset_btn, - process_id = process_id, - model_id = model_id, - model_strength = model_strength, - preview_process = preview_process, - preview_btn = btn_preview, - image_upload = image_upload, - image_reuse = image_reuse, - image_preview = image_preview, - control_start = control_start, - control_end = control_end, - extra_controls = extra_controls, - ) - ) - if i == 0: - units[-1].enabled = True # enable first unit in group - num_controlnet_units.change(fn=helpers.display_units, inputs=[num_controlnet_units], outputs=controlnetxs_ui_units) - - with gr.Tab('Lite') as _tab_lite: - gr.HTML('Control LLLite') - with gr.Row(): - extra_controls = [ - ] - num_lite_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) - lite_ui_units = [] # list of hidable accordions - for i in range(max_units): - enabled = True if i==0 else False - with gr.Accordion(f'Control-LLLite unit {i+1}', visible= i < num_lite_units.value, elem_classes='control-unit') as unit_ui: - with gr.Row(): - enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') - process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') - model_id = gr.Dropdown(label="Model", choices=lite.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') - ui_common.create_refresh_button(model_id, lite.list_models, lambda: {"choices": lite.list_models(refresh=True)}, f'lite_models_{i}_refresh') - model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') - reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'lite_unit-{i}-reset') - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'lite_unit-{i}-upload') - image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'lite_unit-{i}-reuse') - image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'lite_unit-{i}-override') - btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'lite_unit-{i}-preview') - lite_ui_units.append(unit_ui) - units.append(unit.Unit( - unit_type = 'lite', - index = i, - enabled = enabled, - result_txt = result_txt, - enabled_cb = enabled_cb, - reset_btn = reset_btn, - process_id = process_id, - model_id = model_id, - model_strength = model_strength, - preview_process = preview_process, - preview_btn = btn_preview, - image_upload = image_upload, - image_reuse = image_reuse, - image_preview = image_preview, - extra_controls = extra_controls, - ) - ) - if i == 0: - units[-1].enabled = True # enable first unit in group - num_lite_units.change(fn=helpers.display_units, inputs=[num_lite_units], outputs=lite_ui_units) - - with gr.Tab('Reference') as _tab_reference: - gr.HTML('ControlNet reference-only control') - with gr.Row(): - extra_controls = [ - gr.Radio(label="Reference context", choices=['Attention', 'Adain', 'Attention Adain'], value='Attention', interactive=True), - gr.Slider(label="Style fidelity", minimum=0.0, maximum=1.0, step=0.05, value=0.5, interactive=True), # prompt vs control importance - gr.Slider(label="Reference query weight", minimum=0.0, maximum=1.0, step=0.05, value=1.0, interactive=True), - gr.Slider(label="Reference adain weight", minimum=0.0, maximum=2.0, step=0.05, value=1.0, interactive=True), - ] - for i in range(1): # can only have one reference unit - enabled = True if i==0 else False - with gr.Accordion(f'Reference unit {i+1}', visible=True, elem_classes='control-unit') as unit_ui: - with gr.Row(): - enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') - model_id = gr.Dropdown(label="Reference", choices=reference.list_models(), value='Reference', visible=False, elem_id=f'control_unit-{i}-model_name') - model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, visible=False, elem_id=f'control_unit-{i}-strength') - reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'reference_unit-{i}-reset') - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'reference_unit-{i}-upload') - image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'reference_unit-{i}-reuse') - image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'reference_unit-{i}-override') - btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'reference_unit-{i}-preview') - units.append(unit.Unit( - unit_type = 'reference', - index = i, - enabled = enabled, - result_txt = result_txt, - enabled_cb = enabled_cb, - reset_btn = reset_btn, - process_id = process_id, - model_id = model_id, - model_strength = model_strength, - preview_process = preview_process, - preview_btn = btn_preview, - image_upload = image_upload, - image_reuse = image_reuse, - image_preview = image_preview, - extra_controls = extra_controls, - ) - ) - if i == 0: - units[-1].enabled = True # enable first unit in group - - with gr.Accordion('Control settings', open=False, elem_classes=['control-settings']) as _tab_settings: - with gr.Group(elem_classes=['processor-group']): - settings = [] - with gr.Accordion('Global', open=True, elem_classes=['processor-settings']): - control_hires = gr.Checkbox(label="Use control during hires", value=shared.opts.control_hires, elem_id='control_hires') - def set_control_hires(value): - shared.opts.control_active = value - control_hires.change(fn=set_control_hires, inputs=[control_hires], outputs=[]) - control_max_units = gr.Slider(label="Maximum units", minimum=1, maximum=10, step=1, value=shared.opts.control_max_units, elem_id='control_max_units') - def set_control_max_units(value): - shared.opts.control_max_units = value - control_max_units.change(fn=set_control_max_units, inputs=[control_max_units], outputs=[]) - control_tiles = gr.Textbox(label="Tiling options", value=shared.opts.control_tiles, elem_id='control_tiles') - def set_control_tiles(value): - shared.opts.control_tiles = value - control_tiles.change(fn=set_control_tiles, inputs=[control_tiles], outputs=[]) - control_move_processor = gr.Checkbox(label="Move processor to CPU after use", value=shared.opts.control_move_processor, elem_id='control_move_processor') - def set_control_move_processor(value): - shared.opts.control_move_processor = value - control_move_processor.change(fn=set_control_move_processor, inputs=[control_move_processor], outputs=[]) - control_unload_processor = gr.Checkbox(label="Unload processor after use", value=shared.opts.control_unload_processor, elem_id='control_unload_processor') - def set_control_unload_processor(value): - shared.opts.control_unload_processor = value - control_unload_processor.change(fn=set_control_unload_processor, inputs=[control_unload_processor], outputs=[]) - - with gr.Accordion('HED', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Scribble", value=False)) - with gr.Accordion('Midas depth', open=True, elem_classes=['processor-settings']): - settings.append(gr.Slider(label="Background threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1)) - settings.append(gr.Checkbox(label="Depth and normal", value=False)) - with gr.Accordion('MLSD', open=True, elem_classes=['processor-settings']): - settings.append(gr.Slider(label="Score threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1)) - settings.append(gr.Slider(label="Distance threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1)) - with gr.Accordion('OpenBody', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Body", value=True)) - settings.append(gr.Checkbox(label="Hands", value=False)) - settings.append(gr.Checkbox(label="Face", value=False)) - with gr.Accordion('PidiNet', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Scribble", value=False)) - settings.append(gr.Checkbox(label="Apply filter", value=False)) - with gr.Accordion('LineArt', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Coarse", value=False)) - with gr.Accordion('Leres Depth', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Boost", value=False)) - settings.append(gr.Slider(label="Near threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.0)) - settings.append(gr.Slider(label="Depth threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.0)) - with gr.Accordion('MediaPipe Face', open=True, elem_classes=['processor-settings']): - settings.append(gr.Slider(label="Max faces", minimum=1, maximum=10, step=1, value=1)) - settings.append(gr.Slider(label="Face confidence", minimum=0.0, maximum=1.0, step=0.01, value=0.5)) - with gr.Accordion('Canny', open=True, elem_classes=['processor-settings']): - settings.append(gr.Slider(label="Low threshold", minimum=0, maximum=1000, step=1, value=100)) - settings.append(gr.Slider(label="High threshold", minimum=0, maximum=1000, step=1, value=200)) - with gr.Accordion('DWPose', open=True, elem_classes=['processor-settings']): - settings.append(gr.Radio(label="Pose Model", choices=['Tiny', 'Medium', 'Large'], value='Tiny')) - settings.append(gr.Slider(label="Pose confidence", minimum=0.0, maximum=1.0, step=0.01, value=0.3)) - with gr.Accordion('SegmentAnything', open=True, elem_classes=['processor-settings']): - settings.append(gr.Radio(label="Segment Model", choices=['Base', 'Large'], value='Base')) - with gr.Accordion('Edge', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Parameter free", value=True)) - settings.append(gr.Radio(label="Edge mode", choices=['edge', 'gradient'], value='edge')) - with gr.Accordion('Zoe Depth', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Gamma corrected", value=False)) - with gr.Accordion('Marigold Depth', open=True, elem_classes=['processor-settings']): - settings.append(gr.Dropdown(label="Color map", choices=['None'] + plt.colormaps(), value='None')) - settings.append(gr.Slider(label="Denoising steps", minimum=1, maximum=99, step=1, value=10)) - settings.append(gr.Slider(label="Ensemble size", minimum=1, maximum=99, step=1, value=10)) - with gr.Accordion('Depth Anything', open=True, elem_classes=['processor-settings']): - settings.append(gr.Dropdown(label="Depth map", choices=['none'] + masking.COLORMAP, value='inferno')) - for setting in settings: - setting.change(fn=processors.update_settings, inputs=settings, outputs=[]) + from modules.ui_control_elements import create_ui_elements + create_ui_elements(units, result_txt, preview_process) with gr.Row(elem_id="control_script_container"): input_script_args = scripts_manager.scripts_current.setup_ui(parent='control', accordion=True) # handlers - for btn in input_buttons: - btn.click(fn=helpers.copy_input, inputs=[input_mode, btn, input_image, input_resize, input_inpaint], outputs=[input_image, input_resize, input_inpaint], _js='controlInputMode') - btn.click(fn=helpers.transfer_input, inputs=[btn], outputs=[input_image, input_resize, input_inpaint] + input_buttons) + # for btn in input_buttons: + # btn.click(fn=helpers.copy_input, inputs=[input_mode, btn, input_image, input_resize, input_inpaint], outputs=[input_image, input_resize, input_inpaint], _js='controlInputMode') + # btn.click(fn=helpers.transfer_input, inputs=[btn], outputs=[input_image, input_resize, input_inpaint] + input_buttons) # hidden button to update gradio control values for u in units: @@ -583,7 +275,8 @@ def create_ui(_blocks: gr.Blocks=None): prompt, negative, styles, steps, sampler_index, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, - cfg_scale, clip_skip, image_cfg_scale, guidance_rescale, pag_scale, pag_adaptive, cfg_end, vae_type, tiling, hidiffusion, + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, + cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, vae_type, tiling, hidiffusion, detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio, resize_mode_before, resize_name_before, resize_context_before, width_before, height_before, scale_by_before, selected_scale_tab_before, @@ -663,14 +356,19 @@ def create_ui(_blocks: gr.Blocks=None): (mask_controls[4], "Mask erode"), (mask_controls[5], "Mask dilate"), (mask_controls[6], "Mask auto"), + # guidance + (guidance_name, "Guidance"), + (guidance_scale, "Guidance scale"), + (guidance_rescale, "Guidance rescale"), + (guidance_start, "Guidance start"), + (guidance_stop, "Guidance stop"), # advanced - (cfg_scale, "Guidance scale"), (cfg_scale, "CFG scale"), (cfg_end, "CFG end"), (clip_skip, "Clip skip"), (image_cfg_scale, "Image CFG scale"), (image_cfg_scale, "Hires CFG scale"), - (guidance_rescale, "CFG rescale"), + (diffusers_guidance_rescale, "CFG rescale"), (vae_type, "VAE type"), (tiling, "Tiling"), (hidiffusion, "HiDiffusion"), @@ -702,8 +400,8 @@ def create_ui(_blocks: gr.Blocks=None): (refiner_prompt, "Refiner prompt"), (refiner_negative, "Refiner negative"), # pag - (pag_scale, "PAG scale"), - (pag_adaptive, "PAG adaptive"), + (pag_scale, "CFG true"), + (pag_adaptive, "CFG adaptive"), # hidden (seed_resize_from_w, "Seed resize from-1"), (seed_resize_from_h, "Seed resize from-2"), diff --git a/modules/ui_control_elements.py b/modules/ui_control_elements.py new file mode 100644 index 000000000..21f7b4ca5 --- /dev/null +++ b/modules/ui_control_elements.py @@ -0,0 +1,321 @@ +import gradio as gr +import matplotlib.pyplot as plt +from modules.control import unit +from modules.control import processors # patrickvonplaten controlnet_aux +from modules.control.units import controlnet # lllyasviel ControlNet +from modules.control.units import xs # vislearn ControlNet-XS +from modules.control.units import lite # vislearn ControlNet-XS +from modules.control.units import t2iadapter # TencentARC T2I-Adapter +from modules.control.units import reference # reference pipeline +from modules import shared, ui_components, ui_symbols, ui_common, masking # pylint: disable=ungrouped-imports +from modules import ui_control_helpers as helpers + + +def create_ui_elements(units, result_txt, preview_process): + max_units = shared.opts.control_max_units + with gr.Accordion('Control elements', open=False, elem_id="control_elements"): + with gr.Tabs(elem_id='control-tabs') as _tabs_control_type: + + with gr.Tab('ControlNet') as _tab_controlnet: + gr.HTML('ControlNet') + with gr.Row(): + extra_controls = [ + gr.Checkbox(label="Guess mode", value=False, scale=3), + ] + num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) + controlnet_ui_units = [] # list of hidable accordions + for i in range(max_units): + enabled = True if i==0 else False + with gr.Accordion(f'ControlNet unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: + with gr.Row(): + enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') + process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') + model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') + ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, f'controlnet_models_{i}_refresh') + control_mode = gr.Dropdown(label="CN Mode", choices=['default'], value='default', visible=False, elem_id=f'control_unit-{i}-mode') + model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=2.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') + control_start = gr.Slider(label="CN Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') + control_end = gr.Slider(label="CN End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') + control_tile = gr.Dropdown(label="CN Tiles", choices=[x.strip() for x in shared.opts.control_tiles.split(',') if 'x' in x], value='1x1', visible=False, elem_id=f'control_unit-{i}-tile') + reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'controlnet_unit-{i}-reset') + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'controlnet_unit-{i}-upload') + image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'controlnet_unit-{i}-reuse') + btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'controlnet_unit-{i}-preview') + image_preview = gr.Image(label="Input", type="pil", height=128, width=128, visible=False, interactive=True, show_label=False, show_download_button=False, container=False, elem_id=f'controlnet_unit-{i}-override') + controlnet_ui_units.append(unit_ui) + units.append(unit.Unit( + unit_type = 'controlnet', + index = i, + enabled = enabled, + result_txt = result_txt, + enabled_cb = enabled_cb, + reset_btn = reset_btn, + process_id = process_id, + model_id = model_id, + model_strength = model_strength, + preview_process = preview_process, + preview_btn = btn_preview, + image_upload = image_upload, + image_reuse = image_reuse, + image_preview = image_preview, + control_start = control_start, + control_end = control_end, + control_mode = control_mode, + control_tile = control_tile, + extra_controls = extra_controls, + ) + ) + if i == 0: + units[-1].enabled = True # enable first unit in group + num_controlnet_units.change(fn=helpers.display_units, inputs=[num_controlnet_units], outputs=controlnet_ui_units) + + with gr.Tab('T2I Adapter') as _tab_t2iadapter: + gr.HTML('T2I-Adapter') + with gr.Row(): + extra_controls = [ + gr.Slider(label="Control factor", minimum=0.0, maximum=1.0, step=0.05, value=1.0, scale=3), + ] + num_adapter_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) + adapter_ui_units = [] # list of hidable accordions + for i in range(max_units): + enabled = True if i==0 else False + with gr.Accordion(f'T2I-Adapter unit {i+1}', visible= i < num_adapter_units.value, elem_classes='control-unit') as unit_ui: + with gr.Row(): + enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') + process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') + model_id = gr.Dropdown(label="Adapter", choices=t2iadapter.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') + ui_common.create_refresh_button(model_id, t2iadapter.list_models, lambda: {"choices": t2iadapter.list_models(refresh=True)}, f'adapter_models_{i}_refresh') + model_strength = gr.Slider(label="T2I Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') + reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'adapter_unit-{i}-reset') + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'adapter_unit-{i}-upload') + image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'adapter_unit-{i}-reuse') + btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'adapter_unit-{i}-preview') + image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'adapter_unit-{i}-override') + adapter_ui_units.append(unit_ui) + units.append(unit.Unit( + unit_type = 't2i adapter', + index = i, + enabled = enabled, + result_txt = result_txt, + enabled_cb = enabled_cb, + reset_btn = reset_btn, + process_id = process_id, + model_id = model_id, + model_strength = model_strength, + preview_process = preview_process, + preview_btn = btn_preview, + image_upload = image_upload, + image_reuse = image_reuse, + image_preview = image_preview, + extra_controls = extra_controls, + ) + ) + if i == 0: + units[-1].enabled = True # enable first unit in group + num_adapter_units.change(fn=helpers.display_units, inputs=[num_adapter_units], outputs=adapter_ui_units) + + with gr.Tab('XS') as _tab_controlnetxs: + gr.HTML('ControlNet XS') + with gr.Row(): + extra_controls = [ + gr.Slider(label="Time embedding mix", minimum=0.0, maximum=1.0, step=0.05, value=0.0, scale=3) + ] + num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) + controlnetxs_ui_units = [] # list of hidable accordions + for i in range(max_units): + enabled = True if i==0 else False + with gr.Accordion(f'ControlNet-XS unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: + with gr.Row(): + enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') + process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') + model_id = gr.Dropdown(label="ControlNet-XS", choices=xs.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') + ui_common.create_refresh_button(model_id, xs.list_models, lambda: {"choices": xs.list_models(refresh=True)}, f'xs_models_{i}_refresh') + model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') + control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') + control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') + reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'controlnetxs_unit-{i}-reset') + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'controlnetxs_unit-{i}-upload') + image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'controlnetxs_unit-{i}-reuse') + btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'controlnetxs_unit-{i}-preview') + image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'controlnetxs_unit-{i}-override') + controlnetxs_ui_units.append(unit_ui) + units.append(unit.Unit( + unit_type = 'xs', + index = i, + enabled = enabled, + result_txt = result_txt, + enabled_cb = enabled_cb, + reset_btn = reset_btn, + process_id = process_id, + model_id = model_id, + model_strength = model_strength, + preview_process = preview_process, + preview_btn = btn_preview, + image_upload = image_upload, + image_reuse = image_reuse, + image_preview = image_preview, + control_start = control_start, + control_end = control_end, + extra_controls = extra_controls, + ) + ) + if i == 0: + units[-1].enabled = True # enable first unit in group + num_controlnet_units.change(fn=helpers.display_units, inputs=[num_controlnet_units], outputs=controlnetxs_ui_units) + + with gr.Tab('Lite') as _tab_lite: + gr.HTML('Control LLLite') + with gr.Row(): + extra_controls = [ + ] + num_lite_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) + lite_ui_units = [] # list of hidable accordions + for i in range(max_units): + enabled = True if i==0 else False + with gr.Accordion(f'Control-LLLite unit {i+1}', visible= i < num_lite_units.value, elem_classes='control-unit') as unit_ui: + with gr.Row(): + enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') + process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') + model_id = gr.Dropdown(label="Model", choices=lite.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') + ui_common.create_refresh_button(model_id, lite.list_models, lambda: {"choices": lite.list_models(refresh=True)}, f'lite_models_{i}_refresh') + model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') + reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'lite_unit-{i}-reset') + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'lite_unit-{i}-upload') + image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'lite_unit-{i}-reuse') + image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'lite_unit-{i}-override') + btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'lite_unit-{i}-preview') + lite_ui_units.append(unit_ui) + units.append(unit.Unit( + unit_type = 'lite', + index = i, + enabled = enabled, + result_txt = result_txt, + enabled_cb = enabled_cb, + reset_btn = reset_btn, + process_id = process_id, + model_id = model_id, + model_strength = model_strength, + preview_process = preview_process, + preview_btn = btn_preview, + image_upload = image_upload, + image_reuse = image_reuse, + image_preview = image_preview, + extra_controls = extra_controls, + ) + ) + if i == 0: + units[-1].enabled = True # enable first unit in group + num_lite_units.change(fn=helpers.display_units, inputs=[num_lite_units], outputs=lite_ui_units) + + with gr.Tab('Reference') as _tab_reference: + gr.HTML('ControlNet reference-only control') + with gr.Row(): + extra_controls = [ + gr.Radio(label="Reference context", choices=['Attention', 'Adain', 'Attention Adain'], value='Attention', interactive=True), + gr.Slider(label="Style fidelity", minimum=0.0, maximum=1.0, step=0.05, value=0.5, interactive=True), # prompt vs control importance + gr.Slider(label="Reference query weight", minimum=0.0, maximum=1.0, step=0.05, value=1.0, interactive=True), + gr.Slider(label="Reference adain weight", minimum=0.0, maximum=2.0, step=0.05, value=1.0, interactive=True), + ] + for i in range(1): # can only have one reference unit + enabled = True if i==0 else False + with gr.Accordion(f'Reference unit {i+1}', visible=True, elem_classes='control-unit') as unit_ui: + with gr.Row(): + enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') + model_id = gr.Dropdown(label="Reference", choices=reference.list_models(), value='Reference', visible=False, elem_id=f'control_unit-{i}-model_name') + model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, visible=False, elem_id=f'control_unit-{i}-strength') + reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'reference_unit-{i}-reset') + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'reference_unit-{i}-upload') + image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'reference_unit-{i}-reuse') + image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'reference_unit-{i}-override') + btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'reference_unit-{i}-preview') + units.append(unit.Unit( + unit_type = 'reference', + index = i, + enabled = enabled, + result_txt = result_txt, + enabled_cb = enabled_cb, + reset_btn = reset_btn, + process_id = process_id, + model_id = model_id, + model_strength = model_strength, + preview_process = preview_process, + preview_btn = btn_preview, + image_upload = image_upload, + image_reuse = image_reuse, + image_preview = image_preview, + extra_controls = extra_controls, + ) + ) + if i == 0: + units[-1].enabled = True # enable first unit in group + + with gr.Accordion('Control settings', open=False, elem_classes=['control-settings']) as _tab_settings: + with gr.Group(elem_classes=['processor-group']): + settings = [] + with gr.Accordion('Global', open=True, elem_classes=['processor-settings']): + control_hires = gr.Checkbox(label="Use control during hires", value=shared.opts.control_hires, elem_id='control_hires') + def set_control_hires(value): + shared.opts.control_active = value + control_hires.change(fn=set_control_hires, inputs=[control_hires], outputs=[]) + control_max_units = gr.Slider(label="Maximum units", minimum=1, maximum=10, step=1, value=shared.opts.control_max_units, elem_id='control_max_units') + def set_control_max_units(value): + shared.opts.control_max_units = value + control_max_units.change(fn=set_control_max_units, inputs=[control_max_units], outputs=[]) + control_tiles = gr.Textbox(label="Tiling options", value=shared.opts.control_tiles, elem_id='control_tiles') + def set_control_tiles(value): + shared.opts.control_tiles = value + control_tiles.change(fn=set_control_tiles, inputs=[control_tiles], outputs=[]) + control_move_processor = gr.Checkbox(label="Move processor to CPU after use", value=shared.opts.control_move_processor, elem_id='control_move_processor') + def set_control_move_processor(value): + shared.opts.control_move_processor = value + control_move_processor.change(fn=set_control_move_processor, inputs=[control_move_processor], outputs=[]) + control_unload_processor = gr.Checkbox(label="Unload processor after use", value=shared.opts.control_unload_processor, elem_id='control_unload_processor') + def set_control_unload_processor(value): + shared.opts.control_unload_processor = value + control_unload_processor.change(fn=set_control_unload_processor, inputs=[control_unload_processor], outputs=[]) + + with gr.Accordion('HED', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Scribble", value=False)) + with gr.Accordion('Midas depth', open=True, elem_classes=['processor-settings']): + settings.append(gr.Slider(label="Background threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1)) + settings.append(gr.Checkbox(label="Depth and normal", value=False)) + with gr.Accordion('MLSD', open=True, elem_classes=['processor-settings']): + settings.append(gr.Slider(label="Score threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1)) + settings.append(gr.Slider(label="Distance threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1)) + with gr.Accordion('OpenBody', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Body", value=True)) + settings.append(gr.Checkbox(label="Hands", value=False)) + settings.append(gr.Checkbox(label="Face", value=False)) + with gr.Accordion('PidiNet', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Scribble", value=False)) + settings.append(gr.Checkbox(label="Apply filter", value=False)) + with gr.Accordion('LineArt', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Coarse", value=False)) + with gr.Accordion('Leres Depth', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Boost", value=False)) + settings.append(gr.Slider(label="Near threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.0)) + settings.append(gr.Slider(label="Depth threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.0)) + with gr.Accordion('MediaPipe Face', open=True, elem_classes=['processor-settings']): + settings.append(gr.Slider(label="Max faces", minimum=1, maximum=10, step=1, value=1)) + settings.append(gr.Slider(label="Face confidence", minimum=0.0, maximum=1.0, step=0.01, value=0.5)) + with gr.Accordion('Canny', open=True, elem_classes=['processor-settings']): + settings.append(gr.Slider(label="Low threshold", minimum=0, maximum=1000, step=1, value=100)) + settings.append(gr.Slider(label="High threshold", minimum=0, maximum=1000, step=1, value=200)) + with gr.Accordion('DWPose', open=True, elem_classes=['processor-settings']): + settings.append(gr.Radio(label="Pose Model", choices=['Tiny', 'Medium', 'Large'], value='Tiny')) + settings.append(gr.Slider(label="Pose confidence", minimum=0.0, maximum=1.0, step=0.01, value=0.3)) + with gr.Accordion('SegmentAnything', open=True, elem_classes=['processor-settings']): + settings.append(gr.Radio(label="Segment Model", choices=['Base', 'Large'], value='Base')) + with gr.Accordion('Edge', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Parameter free", value=True)) + settings.append(gr.Radio(label="Edge mode", choices=['edge', 'gradient'], value='edge')) + with gr.Accordion('Zoe Depth', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Gamma corrected", value=False)) + with gr.Accordion('Marigold Depth', open=True, elem_classes=['processor-settings']): + settings.append(gr.Dropdown(label="Color map", choices=['None'] + plt.colormaps(), value='None')) + settings.append(gr.Slider(label="Denoising steps", minimum=1, maximum=99, step=1, value=10)) + settings.append(gr.Slider(label="Ensemble size", minimum=1, maximum=99, step=1, value=10)) + with gr.Accordion('Depth Anything', open=True, elem_classes=['processor-settings']): + settings.append(gr.Dropdown(label="Depth map", choices=['none'] + masking.COLORMAP, value='inferno')) + for setting in settings: + setting.change(fn=processors.update_settings, inputs=settings, outputs=[]) diff --git a/modules/ui_control_helpers.py b/modules/ui_control_helpers.py index a04512863..05e31bf1c 100644 --- a/modules/ui_control_helpers.py +++ b/modules/ui_control_helpers.py @@ -24,7 +24,7 @@ def initialize(): from modules.control.units import xs # vislearn ControlNet-XS from modules.control.units import lite # vislearn ControlNet-XS from modules.control.units import t2iadapter # TencentARC T2I-Adapter - shared.log.debug(f'UI initialize: control models="{shared.opts.control_dir}"') + shared.log.debug(f'UI initialize: tab=control models="{shared.opts.control_dir}"') controlnet.cache_dir = os.path.join(shared.opts.control_dir, 'controlnet') xs.cache_dir = os.path.join(shared.opts.control_dir, 'xs') lite.cache_dir = os.path.join(shared.opts.control_dir, 'lite') @@ -77,7 +77,7 @@ def get_video(filepath: str): def select_input(input_mode, input_image, init_image, init_type, input_resize, input_inpaint, input_video, input_batch, input_folder): global busy, input_source, input_init, input_mask # pylint: disable=global-statement busy = True - if input_mode == 'Select': + if input_mode == 'Image': selected_input = input_image elif input_mode == 'Outpaint': selected_input = input_resize @@ -159,16 +159,16 @@ def copy_input(mode_from, mode_to, input_image, input_resize, input_inpaint): if mode_from == mode_to: return [gr.update(), gr.update(), gr.update()] - elif mode_to == 'Select': + elif mode_to == 'Image': return [getimg(input_resize) if mode_from == 'Outpaint' else getimg(input_inpaint), None, None] elif mode_to == 'Inpaint': - return [None, None, getimg(input_image) if mode_from == 'Select' else getimg(input_resize)] + return [None, None, getimg(input_image) if mode_from == 'Image' else getimg(input_resize)] elif mode_to == 'Outpaint': - return [None, getimg(input_image) if mode_from == 'Select' else getimg(input_inpaint), None] + return [None, getimg(input_image) if mode_from == 'Image' else getimg(input_inpaint), None] else: shared.log.error(f'Control transfer unknown input: from={mode_from} to={mode_to}') return [gr.update(), gr.update(), gr.update()] def transfer_input(dst): - return [gr.update(visible=dst=='Select'), gr.update(visible=dst=='Outpaint'), gr.update(visible=dst=='Inpaint'), gr.update(interactive=dst!='Select'), gr.update(interactive=dst!='Inpaint'), gr.update(interactive=dst!='Outpaint')] + return [gr.update(visible=dst=='Image'), gr.update(visible=dst=='Outpaint'), gr.update(visible=dst=='Inpaint'), gr.update(interactive=dst!='Image'), gr.update(interactive=dst!='Inpaint'), gr.update(interactive=dst!='Outpaint')] diff --git a/modules/ui_docs.py b/modules/ui_docs.py index d8b4e3dce..8b1381ce8 100644 --- a/modules/ui_docs.py +++ b/modules/ui_docs.py @@ -275,6 +275,7 @@ def create_ui_docs(): def create_ui(): + log.debug('UI initialize: tab=info') with gr.Tabs(elem_id="tabs_info"): with gr.TabItem("Docs", id="docs", elem_id="system_tab_docs"): create_ui_docs() diff --git a/modules/ui_extensions.py b/modules/ui_extensions.py index 28d792393..43158211a 100644 --- a/modules/ui_extensions.py +++ b/modules/ui_extensions.py @@ -430,6 +430,7 @@ def create_html(search_text, sort_column): def create_ui(): + shared.log.debug('UI initialize: tab=extensions') extensions_disable_all = gr.Radio(label="Disable all extensions", choices=["none", "user", "all"], value=shared.opts.disable_all_extensions, elem_id="extensions_disable_all", visible=False) extensions_disabled_list = gr.Textbox(elem_id="extensions_disabled_list", visible=False, container=False) extensions_update_list = gr.Textbox(elem_id="extensions_update_list", visible=False, container=False) diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 645dab049..c60c04c86 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -828,6 +828,10 @@ def create_ui(container, button_parent, tabname, skip_indexing = False): if is_valid: stat_size, stat_mtime = modelstats.stat(item.filename) + if hasattr(item, 'size') and item.size > 0: + stat_size = item.size + if hasattr(item, 'mtime') and item.mtime is not None: + stat_mtime = item.mtime desc = item.description fullinfo = shared.readfile(os.path.splitext(item.filename)[0] + '.json', silent=True) if 'modelVersions' in fullinfo: # sanitize massive objects diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index 1d320f760..6e6fcc197 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -2,11 +2,13 @@ import os import html import json import concurrent +from datetime import datetime from modules import shared, ui_extra_networks, sd_models, modelstats, paths version_map = { "QwenEdit": "Qwen", + "QwenEditPlus": "Qwen", "Flux.1 D": "Flux", "Flux.1 S": "Flux", "FluxKontext": "Flux", @@ -14,6 +16,8 @@ version_map = { "SDXL Hyper": "SD XL", "StableDiffusion3": "SD 3", "StableDiffusionXL": "SD XL", + "WanToVideo": "Wan", + "WanVACE": "Wan", } class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): @@ -28,6 +32,7 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): def reference_downloaded(url): url = url.split('@')[0] if '@' in url else 'Diffusers/' + url + url = url.split('+')[0] if '+' in url else url return any(model.endswith(url) for model in existing) if not shared.opts.sd_checkpoint_autodownload or not shared.opts.extra_network_reference_enable: @@ -42,8 +47,20 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): continue preview = v.get('preview', v['path']) preview_file = self.find_preview_file(os.path.join(paths.reference_path, preview)) - _size, mtime = modelstats.stat(preview_file) name = os.path.normpath(os.path.join(paths.reference_path, k)).replace('\\', '/') + size = int(float(v.get('size', 0)) * 1024 * 1024 * 1024) + mtime = v.get('date', None) + if mtime is None: + _size, mtime = modelstats.stat(preview_file) + else: + try: + mtime = datetime.strptime(mtime, '%Y %B') # 2025 January + except Exception: + _size, mtime = modelstats.stat(preview_file) + if len(v.get("subfolder", "")) > 0: + path = f'{v.get("path", "")}+{v.get("subfolder", "")}' + else: + path = f'{v.get("path", "")}' yield { "type": 'Model', "name": name, @@ -51,10 +68,10 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): "filename": url, "preview": self.find_preview(os.path.join(paths.reference_path, preview)), "local_preview": preview_file, - "onclick": '"' + html.escape(f"selectReference({json.dumps(url)})") + '"', + "onclick": '"' + html.escape(f"selectReference({json.dumps(path)})") + '"', "hash": None, "mtime": mtime, - "size": 0, + "size": size, "info": {}, "metadata": {}, "description": v.get('desc', ''), diff --git a/modules/ui_extra_networks_lora.py b/modules/ui_extra_networks_lora.py index d390fe914..6b9705674 100644 --- a/modules/ui_extra_networks_lora.py +++ b/modules/ui_extra_networks_lora.py @@ -85,7 +85,7 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): "metadata": json.dumps(l.metadata, indent=4) if l.metadata else None, "mtime": mtime, "size": size, - "version": version.get("baseModel", l.sd_version) if info else l.sd_version, + "version": version.get("baseModel", l.sd_version), "info": info, "description": self.find_description(l.filename, info), "tags": self.get_tags(l, info, version), diff --git a/modules/ui_gallery.py b/modules/ui_gallery.py index e6cb3f297..283b720a8 100644 --- a/modules/ui_gallery.py +++ b/modules/ui_gallery.py @@ -54,6 +54,7 @@ def read_media(fn): def create_ui(): + shared.log.debug('UI initialize: tab=gallery') with gr.Blocks() as tab: with gr.Row(elem_id='tab-gallery-sort-buttons'): sort_buttons = [] diff --git a/modules/ui_guidance.py b/modules/ui_guidance.py new file mode 100644 index 000000000..a6ea46391 --- /dev/null +++ b/modules/ui_guidance.py @@ -0,0 +1,128 @@ +import gradio as gr +from modules import shared +from modules import ui_symbols, ui_components + + +guiders = ['Default', 'CFG', 'Zero', 'PAG', 'APG', 'SLG', 'SEG', 'TCFG', 'FDG'] + + +def create_guidance_inputs(tab): + with gr.Accordion(open=False, label='Guidance', elem_id=f"{tab}_guidance", elem_classes=["small-accordion"]): + with gr.Group(): + + with gr.Row(elem_id=f"{tab}_guider_row", elem_classes=['flexbox'], visible=shared.opts.model_modular_enable): + guidance_name = gr.Dropdown(choices=guiders, value='Default', label='Guider', elem_id=f"{tab}_guider") + guidance_btn = ui_components.ToolButton(value=ui_symbols.book, elem_id=f"{tab}_guider_docs") + guidance_btn.click(fn=None, _js='getGuidanceDocs', inputs=[guidance_name], outputs=[]) + with gr.Row(visible=shared.opts.model_modular_enable): + guidance_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='_Guidance scale', value=6.0, elem_id=f"{tab}_guidance_scale") + guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='_Guidance rescale', value=0.0, elem_id=f"{tab}_guidance_rescale") + with gr.Row(visible=shared.opts.model_modular_enable): + guidance_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='_Guidance start', value=0.0, elem_id=f"{tab}_guidance_start") + guidance_stop = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='_Guidance stop', value=1.0, elem_id=f"{tab}_guidance_stop") + guidance_args = [guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop] + + lsc_group = gr.Accordion(open=False, label='Layer skip guidance', elem_classes=["small-accordion"], visible=shared.opts.model_modular_enable) + with lsc_group: + with gr.Row(): + guidance_lsc_enabled = gr.Checkbox(label='Enable LayerSkipConfig', value=False) + guidance_lsc_label = gr.Label(value='LSC: LayerSkipConfig', elem_id=f"{tab}_lsc_label", visible=False) + guidance_lsc_btn = ui_components.ToolButton(value=ui_symbols.book, elem_id=f"{tab}_lsc_docs", elem_classes=["guidance-docs"]) + guidance_lsc_btn.click(fn=None, _js='getGuidanceDocs', inputs=[guidance_lsc_label], outputs=[]) + with gr.Row(): + guidance_lsc_indices = gr.Textbox(label='LSC layer indices', value='1, 2, 3', placeholder='Comma-separated layer indices to skip') + with gr.Row(): + guidance_lsc_fqn = gr.Textbox(label='LSC fully qualified name', value='transformer_blocks', placeholder='Fully qualified name of the layer stack') + with gr.Row(): + guidance_lsc_skip_attention = gr.Checkbox(label='LSC skip attention blocks', value=True) + guidance_lsc_skip_ff = gr.Checkbox(label='LSC skip feed-forward blocks', value=True) + guidance_lsc_skip_attention_scores = gr.Checkbox(label='LSC skip attention scores', value=False) + with gr.Row(): + guidance_lsc_dropout = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='LSC dropout rate', value=1.0) + lsc_args = [guidance_lsc_enabled, guidance_lsc_indices, guidance_lsc_fqn, guidance_lsc_skip_attention, guidance_lsc_skip_ff, guidance_lsc_skip_attention_scores, guidance_lsc_dropout] + + auto_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with auto_group: + guidance_auto_dropout = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='AutoGuidance dropout', value=0.1) + guidance_auto_layers = gr.Textbox(label='AutoGuidance layers', value='7, 8, 9', placeholder='Comma-separated layer indices, e.g. 7,8,9') + guidance_auto_config = gr.Dropdown(choices=[None, 'config1', 'config2'], value=None, label='AutoGuidance config') + guidance_auto_args = [guidance_auto_dropout, guidance_auto_layers, guidance_auto_config] + + zero_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with zero_group: + guidance_zero_init_steps = gr.Slider(minimum=0, maximum=10, step=1, label='ZeroStar init steps', value=1) + guidance_zero_args = [guidance_zero_init_steps] + + pag_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with pag_group: + guidance_pag_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.05, label='PAG scale', value=2.8) + guidance_pag_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='PAG start', value=0.01) + guidance_pag_stop = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='PAG stop', value=0.2) + guidance_pag_layers = gr.Textbox(label='PAG layers', value='7, 8, 9', placeholder='Comma-separated layer indices, e.g. 7,8,9') + guidance_pag_config = gr.Dropdown(choices=[None, 'config1', 'config2'], value=None, label='PAG config') + guidance_pag_args = [guidance_pag_scale, guidance_pag_start, guidance_pag_stop, guidance_pag_layers, guidance_pag_config] + + apg_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with apg_group: + guidance_apg_momentum = gr.Slider(minimum=-1.0, maximum=1.0, step=0.05, label='APG momentum', value=-1.0) + guidance_apg_rescale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='APG rescale', value=15.0) + guidance_apg_args = [guidance_apg_momentum, guidance_apg_rescale] + + slg_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with slg_group: + guidance_slg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='SLG scale', value=2.8) + guidance_slg_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='SLG start', value=0.01) + guidance_slg_stop = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='SLG stop', value=0.2) + guidance_slg_layers = gr.Textbox(label='SLG layers', value='7, 8, 9', placeholder='Comma-separated layer indices, e.g. 7,8,9') + guidance_slg_config = gr.Dropdown(choices=[None, 'config1', 'config2'], value=None, label='SLG config') + guidance_slg_args = [guidance_slg_scale, guidance_slg_start, guidance_slg_stop, guidance_slg_layers, guidance_slg_config] + + seg_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with seg_group: + guidance_seg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='SEG scale', value=3.0) + guidance_seg_blur_sigma = gr.Number(label='SEG blur sigma', value=9999999.0) + guidance_seg_blur_threshold_inf = gr.Number(label='SEG blur threshold inf', value=9999.0) + guidance_seg_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='SEG start', value=0.0) + guidance_seg_stop = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='SEG stop', value=1.0) + guidance_seg_layers = gr.Textbox(label='SEG layers', value='7, 8, 9', placeholder='Comma-separated layer indices, e.g. 7,8,9') + guidance_seg_config = gr.Dropdown(choices=[None, 'config1', 'config2'], value=None, label='SEG config') + guidance_seg_args = [guidance_seg_scale, guidance_seg_blur_sigma, guidance_seg_blur_threshold_inf, guidance_seg_start, guidance_seg_stop, guidance_seg_layers, guidance_seg_config] + + tcfg_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with tcfg_group: + pass + + fdg_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with fdg_group: + guidance_fdg_scales = gr.Textbox(label='FDG scales', value='10.0, 5.0', placeholder='Comma-separated scales, e.g. 10.0,5.0') + guidance_fdg_weights = gr.Textbox(label='FDG weights', value='1.0', placeholder='Single float or comma-separated weights, e.g. 1.0 or 1.0,0.5') + guidance_fdg_rescale_space = gr.Dropdown(choices=['data', 'freq'], value='data', label='FDG rescale space') + guidance_fdg_args = [guidance_fdg_scales, guidance_fdg_weights, guidance_fdg_rescale_space] + + def adv_visibility(guidance_name): + return [ + gr.update(visible=guidance_name.startswith('Auto')), + gr.update(visible=guidance_name.startswith('Zero')), + gr.update(visible=guidance_name.startswith('PAG')), + gr.update(visible=guidance_name.startswith('APG')), + gr.update(visible=guidance_name.startswith('SLG')), + gr.update(visible=guidance_name.startswith('SEG')), + gr.update(visible=guidance_name.startswith('TCFG')), + gr.update(visible=guidance_name.startswith('FDG')), + ] + guidance_name.change(fn=adv_visibility, inputs=[guidance_name], outputs=[auto_group, zero_group, pag_group, apg_group, slg_group, seg_group, tcfg_group, fdg_group]) + + gr.HTML(value='

Fallback guidance

', visible=shared.opts.model_modular_enable, elem_id=f"{tab}_guidance_note") + with gr.Row(elem_id=f"{tab}_cfg_row", elem_classes=['flexbox']): + cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Guidance scale', value=6.0, elem_id=f"{tab}_cfg_scale") + cfg_end = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='Guidance end', value=1.0, elem_id=f"{tab}_cfg_end") + with gr.Row(): + diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.0, elem_id=f"{tab}_image_cfg_rescale") + image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Refine guidance', value=6.0, elem_id=f"{tab}_image_cfg_scale") + with gr.Row(): + diffusers_pag_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.05, label='Attention guidance', value=0.0, elem_id=f"{tab}_pag_scale") + diffusers_pag_adaptive = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Adaptive scaling', value=0.5, elem_id=f"{tab}_pag_adaptive") + + _modular_args = guidance_args + lsc_args + guidance_auto_args + guidance_zero_args + guidance_pag_args + guidance_apg_args + guidance_slg_args + guidance_seg_args + guidance_fdg_args + standard_args = [cfg_scale, image_cfg_scale, diffusers_guidance_rescale, diffusers_pag_scale, diffusers_pag_adaptive, cfg_end] + return guidance_args + standard_args diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index c1597321d..a22e66901 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -1,11 +1,11 @@ -import os -from PIL import Image import gradio as gr -from modules.call_queue import wrap_gradio_gpu_call, wrap_queued_call -from modules import timer, shared, ui_common, ui_sections, generation_parameters_copypaste, processing_vae +from modules import timer, shared, call_queue, generation_parameters_copypaste, processing_vae +from modules import ui_common, ui_sections, ui_guidance def process_interrogate(mode, ii_input_files, ii_input_dir, ii_output_dir, *ii_singles): + import os + from PIL import Image from modules.interrogate.interrogate import interrogate mode = int(mode) if mode in {0, 1, 3, 4}: @@ -33,7 +33,7 @@ def process_interrogate(mode, ii_input_files, ii_input_dir, ii_output_dir, *ii_s def create_ui(): - shared.log.debug('UI initialize: img2img') + shared.log.debug('UI initialize: tab=img2img') import modules.img2img # pylint: disable=redefined-outer-name modules.scripts_manager.scripts_current = modules.scripts_manager.scripts_img2img modules.scripts_manager.scripts_img2img.initialize_scripts(is_img2img=True, is_control=False) @@ -132,9 +132,10 @@ def create_ui(): denoising_strength = gr.Slider(minimum=0.00, maximum=0.99, step=0.01, label='Denoising strength', value=0.30, elem_id="img2img_denoising_strength") refiner_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.0, elem_id="img2img_refiner_start") - vae_type, tiling, hidiffusion, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_sections.create_advanced_inputs('img2img') + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_guidance.create_guidance_inputs('img2img') + vae_type, tiling, hidiffusion, clip_skip = ui_sections.create_advanced_inputs('img2img') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('img2img') - enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, hr_refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') + enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, hr_refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('img2img') detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution = shared.yolo.ui('img2img') # with gr.Group(elem_id="inpaint_controls", visible=False) as inpaint_controls: @@ -176,8 +177,8 @@ def create_ui(): vae_type, tiling, hidiffusion, detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, batch_count, batch_size, - cfg_scale, image_cfg_scale, - diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, + cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, refiner_start, clip_skip, denoising_strength, @@ -193,7 +194,7 @@ def create_ui(): override_settings, ] img2img_dict = dict( - fn=wrap_gradio_gpu_call(modules.img2img.img2img, extra_outputs=[None, '', ''], name='Image'), + fn=call_queue.wrap_gradio_gpu_call(modules.img2img.img2img, extra_outputs=[None, '', ''], name='Image'), _js="submit_img2img", inputs= img2img_args + img2img_script_inputs, outputs=[ @@ -228,8 +229,8 @@ def create_ui(): ) interrogate_btn.click(fn=lambda *args: process_interrogate(*args), **interrogate_args) - img2img_token_button.click(fn=wrap_queued_call(ui_common.update_token_counter), inputs=[img2img_prompt], outputs=[img2img_token_counter], show_progress = False) - img2img_negative_token_button.click(fn=wrap_queued_call(ui_common.update_token_counter), inputs=[img2img_negative_prompt], outputs=[img2img_negative_token_counter], show_progress = False) + img2img_token_button.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[img2img_prompt], outputs=[img2img_token_counter], show_progress = False) + img2img_negative_token_button.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[img2img_negative_prompt], outputs=[img2img_negative_token_counter], show_progress = False) ui_extra_networks.setup_ui(extra_networks_ui_img2img, img2img_gallery) img2img_paste_fields = [ @@ -253,8 +254,13 @@ def create_ui(): (seed, "Seed"), (subseed, "Variation seed"), (subseed_strength, "Variation strength"), + # guidance + (guidance_name, "Guidance"), + (guidance_scale, "Guidance scale"), + (guidance_rescale, "Guidance rescale"), + (guidance_start, "Guidance start"), + (guidance_stop, "Guidance stop"), # advanced - (cfg_scale, "Guidance scale"), (cfg_scale, "CFG scale"), (cfg_end, "CFG end"), (image_cfg_scale, "Image CFG scale"), @@ -292,8 +298,8 @@ def create_ui(): (refiner_prompt, "refiner prompt"), (refiner_negative, "Refiner negative"), # pag - (pag_scale, "PAG scale"), - (pag_adaptive, "PAG adaptive"), + (pag_scale, "CFG true"), + (pag_adaptive, "CFG adaptive"), # inpaint (mask_blur, "Mask blur"), (mask_alpha, "Mask alpha"), diff --git a/modules/ui_loadsave.py b/modules/ui_loadsave.py index a8e3dce9b..3b6484e06 100644 --- a/modules/ui_loadsave.py +++ b/modules/ui_loadsave.py @@ -35,7 +35,6 @@ class UiLoadsave: saved_value = self.ui_settings.get(key, None) self.ui_defaults[key] = getattr(obj, field) if saved_value is None: - # self.ui_settings[key] = getattr(obj, field) pass elif condition and not condition(saved_value): pass @@ -126,15 +125,37 @@ class UiLoadsave: writefile(current_ui_settings, self.filename) def dump_defaults(self): - """saves default values to a file unless the file is present and there was an error loading default values at start""" if os.path.exists(self.filename): return self.write_to_file(self.ui_settings) + def iter_all(self, values): + updates = [] + for i, name in enumerate(self.component_mapping): + component = self.component_mapping[name] + choices = getattr(component, 'choices', None) + if type(choices) is list and len(choices) > 0: # fix gradio radio button choices being tuples + if type(choices[0]) is tuple: + choices = [c[0] for c in choices] + new_value = values[i] + if isinstance(new_value, int) and choices: + if new_value >= len(choices): + updates.append(None) + new_value = choices[new_value] + old_value = self.ui_settings.get(name, None) + default_value = self.ui_defaults.get(name, '') + if old_value == new_value: + updates.append(None) + elif old_value is None and (new_value == '' or new_value == []): + updates.append(None) + elif (new_value == default_value) and (old_value is None): + updates.append(None) + else: + updates.append((name, old_value, new_value, default_value)) + return updates + def iter_changes(self, values): for i, name in enumerate(self.component_mapping): - # if '__init__' in name: - # continue component = self.component_mapping[name] choices = getattr(component, 'choices', None) if type(choices) is list and len(choices) > 0: # fix gradio radio button choices being tuples @@ -201,16 +222,23 @@ class UiLoadsave: def ui_apply(self, *values): num_changed = 0 + num_unchanged = 0 current_ui_settings = self.read_from_file() - for name, old_value, new_value, default_value in self.iter_changes(values): - component = self.component_mapping[name] - errors.log.debug(f'Settings: name={name} component={component} old={old_value} default={default_value} new={new_value}') - num_changed += 1 - current_ui_settings[name] = new_value + for x in self.iter_all(values): + if x is None: + num_unchanged += 1 + else: + name, old_value, new_value, default_value = x + component = self.component_mapping[name] + errors.log.debug(f'Settings: name={name} component={component} old={old_value} default={default_value} new={new_value}') + num_changed += 1 + current_ui_settings[name] = new_value + # what = name.split('/')[-1] + # setattr(component, what, new_value) if num_changed == 0: return "No changes" self.write_to_file(current_ui_settings) - errors.log.info(f'UI defaults saved: {self.filename}') + errors.log.info(f'UI defaults saved: {self.filename} changes={num_changed} unchanged={num_unchanged}') return f"Wrote {num_changed} changes" def ui_submenu_apply(self, items): diff --git a/modules/ui_models.py b/modules/ui_models.py index bfb876372..89e5697ab 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -12,6 +12,7 @@ extra_ui = [] def create_ui(): + log.debug('UI initialize: tab=models') dummy_component = gr.Label(visible=False) with gr.Row(elem_id="models_tab"): with gr.Column(elem_id='models_output_container', scale=1): @@ -64,12 +65,23 @@ def create_ui(): return [html, meta] with gr.Row(): - gr.HTML('

Analyze currently loaded model

') - with gr.Row(): - model_analyze = gr.Button(value="Analyze", variant='primary') + model_analyze = gr.Button(value="Analyze model", variant='primary') with gr.Row(): model_desc = gr.HTML(value="", elem_id="model_desc") - with gr.Row(): + with gr.Accordion(label="Save model", open=False): + with gr.Row(): + save_name = gr.Textbox(label="Model name", placeholder="Model name to save as") + with gr.Row(): + save_path = gr.Textbox(label="Model base path", placeholder="Path to save model to", value=opts.diffusers_dir) + with gr.Row(): + save_shard = gr.Textbox(label="Max shard size", placeholder="Maximum shard size", value="10GB") + save_overwrite = gr.Checkbox(label="Overwrite existing", value=False) + with gr.Row(): + save_result = gr.HTML(value="", elem_id="model_save_outcome") + with gr.Row(): + model_save = gr.Button(value="Save model", variant='primary') + model_save.click(fn=sd_models.save_model, inputs=[save_name, save_path, save_shard, save_overwrite], outputs=[save_result]) + with gr.Accordion(label="Metadata", open=False): model_meta = gr.JSON(label="Metadata", value={}, elem_id="model_meta") model_analyze.click(fn=analyze, inputs=[], outputs=[model_desc, model_meta]) diff --git a/modules/ui_postprocessing.py b/modules/ui_postprocessing.py index b575fa67a..5ed90446a 100644 --- a/modules/ui_postprocessing.py +++ b/modules/ui_postprocessing.py @@ -16,6 +16,7 @@ def submit_process(tab_index, extras_image, image_batch, extras_batch_input_dir, def create_ui(): + shared.log.debug('UI initialize: tab=process') tab_index = gr.State(value=0) # pylint: disable=abstract-class-instantiated with gr.Row(equal_height=False, variant='compact', elem_classes="extras", elem_id="extras_tab"): with gr.Column(variant='compact'): diff --git a/modules/ui_sections.py b/modules/ui_sections.py index 07ce95b3c..683681cd5 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -146,26 +146,9 @@ def create_video_inputs(tab:str, show_always:bool=False): return video_type, video_duration, video_loop, video_pad, video_interpolate -def create_cfg_inputs(tab): - with gr.Row(elem_id=f"{tab}_cfg_row", elem_classes=['flexbox']): - cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Guidance scale', value=6.0, elem_id=f"{tab}_cfg_scale") - cfg_end = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='Guidance end', value=1.0, elem_id=f"{tab}_cfg_end") - return cfg_scale, cfg_end - - -def create_advanced_inputs(tab, base=True): +def create_advanced_inputs(tab): with gr.Accordion(open=False, label="Advanced", elem_id=f"{tab}_advanced", elem_classes=["small-accordion"]): with gr.Group(): - if base: - cfg_scale, cfg_end = create_cfg_inputs(tab) - else: - cfg_scale, cfg_end = None, None - with gr.Row(): - image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Refine guidance', value=6.0, elem_id=f"{tab}_image_cfg_scale") - diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Rescale guidance', value=0.0, elem_id=f"{tab}_image_cfg_rescale") - with gr.Row(): - diffusers_pag_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.05, label='Attention guidance', value=0.0, elem_id=f"{tab}_pag_scale") - diffusers_pag_adaptive = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Adaptive scaling', value=0.5, elem_id=f"{tab}_pag_adaptive") with gr.Row(): clip_skip = gr.Slider(label='CLiP skip', value=1, minimum=0, maximum=12, step=0.1, elem_id=f"{tab}_clip_skip", interactive=shared.opts.clip_skip_enabled) with gr.Row(elem_id=f"{tab}_vae_options"): @@ -173,7 +156,7 @@ def create_advanced_inputs(tab, base=True): with gr.Row(elem_id=f"{tab}_advanced_options"): tiling = gr.Checkbox(label='Texture tiling', value=False, elem_id=f"{tab}_tiling") hidiffusion = gr.Checkbox(label='HiDiffusion', value=False, elem_id=f"{tab}_hidiffusion") - return vae_type, tiling, hidiffusion, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, diffusers_pag_scale, diffusers_pag_adaptive, cfg_end + return vae_type, tiling, hidiffusion, clip_skip def create_correction_inputs(tab): diff --git a/modules/ui_settings.py b/modules/ui_settings.py index 9b24464cc..673e45941 100644 --- a/modules/ui_settings.py +++ b/modules/ui_settings.py @@ -24,7 +24,7 @@ def apply_setting(key, value): if key in shared.opts.disable_apply_metadata: gr.update() if key == "sd_model_checkpoint": - ckpt_info = sd_models.get_closet_checkpoint_match(value) + ckpt_info = sd_models.get_closest_checkpoint_match(value) if ckpt_info is not None: value = ckpt_info.title else: @@ -172,6 +172,7 @@ def run_settings_single(value, key, progress=False): def create_ui(): + shared.log.debug('UI initialize: tab=settings') global text_settings # pylint: disable=global-statement text_settings = gr.Textbox(elem_id="settings_json", elem_classes=["settings_json"], value=lambda: shared.opts.dumpjson(), visible=False) with gr.Row(elem_id="system_row"): @@ -226,7 +227,7 @@ def create_ui(): quicksettings_list.append((key, item)) components.append(dummy_component) else: - with gr.Row(elem_id=f"settings_section_row_{section_id}"): # only so we can add dirty indicator at the start of the row + with gr.Row(elem_id=f"settings_section_row_{section_id}", elem_classes=["settings_section"]): # only so we can add dirty indicator at the start of the row component = create_setting_component(key) shared.settings_components[key] = component current_items.append(key) diff --git a/modules/ui_txt2img.py b/modules/ui_txt2img.py index fe9a19c50..f3e108666 100644 --- a/modules/ui_txt2img.py +++ b/modules/ui_txt2img.py @@ -1,11 +1,10 @@ import gradio as gr -from modules.call_queue import wrap_gradio_gpu_call, wrap_queued_call -from modules import timer, shared, ui_common, ui_sections, generation_parameters_copypaste, processing_vae, images -from modules.ui_components import ToolButton # pylint: disable=unused-import +from modules import timer, shared, call_queue, generation_parameters_copypaste, processing_vae, images +from modules import ui_common, ui_sections, ui_guidance def create_ui(): - shared.log.debug('UI initialize: txt2img') + shared.log.debug('UI initialize: tab=txt2img') import modules.txt2img # pylint: disable=redefined-outer-name modules.scripts_manager.scripts_current = modules.scripts_manager.scripts_txt2img modules.scripts_manager.scripts_txt2img.initialize_scripts(is_img2img=False, is_control=False) @@ -27,14 +26,14 @@ def create_ui(): width, height = ui_sections.create_resolution_inputs('txt2img') batch_count, batch_size = ui_sections.create_batch_inputs('txt2img', accordion=False) - cfg_scale, cfg_end = ui_sections.create_cfg_inputs('txt2img') steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "txt2img") with gr.Group(elem_classes="settings-accordion"): with gr.Accordion(open=False, label="Samplers", elem_classes=["small-accordion"], elem_id="txt2img_sampler_group"): ui_sections.create_sampler_options('txt2img') seed, reuse_seed, subseed, reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w = ui_sections.create_seed_inputs('txt2img') - vae_type, tiling, hidiffusion, _cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, _cfg_end = ui_sections.create_advanced_inputs('txt2img', base=False) + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_guidance.create_guidance_inputs('txt2img') + vae_type, tiling, hidiffusion, clip_skip = ui_sections.create_advanced_inputs('txt2img') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('txt2img') enable_hr, hr_sampler_index, denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution = shared.yolo.ui('txt2img') @@ -57,6 +56,7 @@ def create_ui(): vae_type, tiling, hidiffusion, detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, batch_count, batch_size, + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, clip_skip, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, @@ -68,7 +68,7 @@ def create_ui(): override_settings, ] txt2img_dict = dict( - fn=wrap_gradio_gpu_call(modules.txt2img.txt2img, extra_outputs=[None, '', ''], name='Text'), + fn=call_queue.wrap_gradio_gpu_call(modules.txt2img.txt2img, extra_outputs=[None, '', ''], name='Text'), _js="submit_txt2img", inputs=txt2img_args + txt2img_script_inputs, outputs=[ @@ -106,8 +106,13 @@ def create_ui(): (seed, "Seed"), (subseed, "Variation seed"), (subseed_strength, "Variation strength"), + # guidance + (guidance_name, "Guidance"), + (guidance_scale, "Guidance scale"), + (guidance_rescale, "Guidance rescale"), + (guidance_start, "Guidance start"), + (guidance_stop, "Guidance stop"), # advanced - (cfg_scale, "Guidance scale"), (cfg_scale, "CFG scale"), (cfg_end, "CFG end"), (clip_skip, "Clip skip"), @@ -145,8 +150,8 @@ def create_ui(): (refiner_prompt, "refiner prompt"), (refiner_negative, "Refiner negative"), # pag - (pag_scale, "PAG scale"), - (pag_adaptive, "PAG adaptive"), + (pag_scale, "CFG true"), + (pag_adaptive, "CFG adaptive"), # hidden (seed_resize_from_w, "Seed resize from-1"), (seed_resize_from_h, "Seed resize from-2"), @@ -156,7 +161,7 @@ def create_ui(): txt2img_bindings = generation_parameters_copypaste.ParamBinding(paste_button=txt2img_paste, tabname="txt2img", source_text_component=txt2img_prompt, source_image_component=None) generation_parameters_copypaste.register_paste_params_button(txt2img_bindings) - txt2img_token_button.click(fn=wrap_queued_call(ui_common.update_token_counter), inputs=[txt2img_prompt], outputs=[txt2img_token_counter], show_progress = False) - txt2img_negative_token_button.click(fn=wrap_queued_call(ui_common.update_token_counter), inputs=[txt2img_negative_prompt], outputs=[txt2img_negative_token_counter], show_progress = False) + txt2img_token_button.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[txt2img_prompt], outputs=[txt2img_token_counter], show_progress = False) + txt2img_negative_token_button.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[txt2img_negative_prompt], outputs=[txt2img_negative_token_counter], show_progress = False) ui_extra_networks.setup_ui(extra_networks_ui, txt2img_gallery) diff --git a/modules/ui_video.py b/modules/ui_video.py index bb9cace94..02c787c6e 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -7,7 +7,7 @@ debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None e def create_ui(): - shared.log.debug('UI initialize: video') + shared.log.debug('UI initialize: tab=video') with gr.Blocks(analytics_enabled=False) as _video_interface: prompt, styles, negative, generate_btn, _reprocess, paste, networks_button, _token_counter, _token_button, _token_counter_negative, _token_button_negative = ui_sections.create_toprow( is_img2img=False, diff --git a/modules/ui_video_vlm.py b/modules/ui_video_vlm.py index 685c6d8d8..c7ce3a229 100644 --- a/modules/ui_video_vlm.py +++ b/modules/ui_video_vlm.py @@ -46,7 +46,6 @@ def enhance_prompt(enable:bool, model:str=None, image=None, prompt:str='', syste system_prompt += system_prompts['nsfw_ok'] if nsfw else system_prompts['nsfw_no'] system_prompt += f" {system_prompts['suffix']} {system_prompts['example']}" shared.log.debug(f'Video prompt enhance: model="{model}" image={image} nsfw={nsfw} prompt="{prompt}"') - # shared.log.trace(f'Video prompt enhance: system="{system_prompt}"') answer = vqa.interrogate(question='', prompt=prompt, system_prompt=system_prompt, image=image, model_name=model, quiet=False) shared.log.debug(f'Video prompt enhance: answer="{answer}"') return answer diff --git a/modules/upscaler.py b/modules/upscaler.py index 3494742e8..ec907ad8a 100644 --- a/modules/upscaler.py +++ b/modules/upscaler.py @@ -79,7 +79,7 @@ class Upscaler: scalers.append(scaler) loaded.append(model_path) # shared.log.debug(f'Upscaler type={self.name} folder="{self.user_path}" model="{model[0]}" path="{model_path}"') - if not os.path.exists(self.user_path): + if self.user_path is None or not os.path.exists(self.user_path): return scalers self.find_folder(self.user_path, scalers, loaded) return scalers @@ -148,7 +148,7 @@ class UpscalerData: scaler: Upscaler = None model: None - def __init__(self, name: str, path: str, upscaler: Upscaler = None, scale: int = 4, model=None): + def __init__(self, name: str, path: str = None, upscaler: Upscaler = None, scale: int = 4, model=None): self.name = name self.data_path = path self.local_data_path = path diff --git a/modules/video.py b/modules/video.py index fe7d88c36..b95ae7aa1 100644 --- a/modules/video.py +++ b/modules/video.py @@ -29,7 +29,7 @@ def save_video_atomic(images, filename, video_type: str = 'none', duration: floa except Exception as e: shared.log.error(f'Save video: cv2: {e}') return - jobid = shared.state.begin('Save video') + savejob = shared.state.begin('Save video') os.makedirs(os.path.dirname(filename), exist_ok=True) if video_type.lower() in ['gif', 'png']: append = images.copy() @@ -57,7 +57,7 @@ def save_video_atomic(images, filename, video_type: str = 'none', duration: floa video_writer.write(img) size = os.path.getsize(filename) shared.log.info(f'Save video: file="{filename}" frames={len(frames)} duration={duration} fourcc={fourcc} size={size}') - shared.state.end(jobid) + shared.state.end(savejob) def save_video(p, images, filename = None, video_type: str = 'none', duration: float = 2.0, loop: bool = False, interpolate: int = 0, scale: float = 1.0, pad: int = 1, change: float = 0.3, sync: bool = False): diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index 6d92db375..14f0eb8e0 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -74,6 +74,12 @@ models = { ], 'LTX Video': [ Model(name='None'), + Model(name='LTXVideo 0.9.8 13B', + url='https://huggingface.co/Lightricks/LTX-Video-0.9.8-13B-distilled', + repo='Lightricks/LTX-Video-0.9.8-13B-distilled', + repo_cls=diffusers.LTXConditionPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), Model(name='LTXVideo 0.9.7 13B', url='https://huggingface.co/Lightricks/LTX-Video-0.9.7-dev', repo='a-r-r-o-w/LTX-Video-0.9.7-diffusers', @@ -147,70 +153,77 @@ models = { url='https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B-Diffusers', repo='Wan-AI/Wan2.2-TI2V-5B-Diffusers', repo_cls=diffusers.WanPipeline, - te_cls=transformers.T5EncoderModel, + te_cls=transformers.UMT5EncoderModel, dit_cls=diffusers.WanTransformer3DModel), Model(name='WAN 2.2 5B I2V', url='https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B-Diffusers', repo='Wan-AI/Wan2.2-TI2V-5B-Diffusers', repo_cls=diffusers.WanImageToVideoPipeline, - te_cls=transformers.T5EncoderModel, + te_cls=transformers.UMT5EncoderModel, dit_cls=diffusers.WanTransformer3DModel), Model(name='WAN 2.2 A14B T2V', url='https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B-Diffusers', repo='Wan-AI/Wan2.2-T2V-A14B-Diffusers', repo_cls=diffusers.WanPipeline, - te_cls=transformers.T5EncoderModel, + te_cls=transformers.UMT5EncoderModel, dit_cls=diffusers.WanTransformer3DModel, dit_folder=("transformer", "transformer_2")), Model(name='WAN 2.2 A14B I2V', url='https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B-Diffusers', repo='Wan-AI/Wan2.2-I2V-A14B-Diffusers', repo_cls=diffusers.WanImageToVideoPipeline, - te_cls=transformers.T5EncoderModel, + te_cls=transformers.UMT5EncoderModel, dit_cls=diffusers.WanTransformer3DModel, dit_folder=("transformer", "transformer_2")), + Model(name='WAN 2.2 14B VACE', + url='https://huggingface.co/linoyts/Wan2.2-VACE-Fun-14B-diffusers', + repo='linoyts/Wan2.2-VACE-Fun-14B-diffusers', + repo_cls=diffusers.WanVACEPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanVACETransformer3DModel, + dit_folder=("transformer", "transformer_2")), Model(name='WAN 2.1 1.3B T2V', url='https://huggingface.co/Wan-AI/Wan2.1-T2V-1.3B-Diffusers', repo='Wan-AI/Wan2.1-T2V-1.3B-Diffusers', repo_cls=diffusers.WanPipeline, - te_cls=transformers.T5EncoderModel, + te_cls=transformers.UMT5EncoderModel, dit_cls=diffusers.WanTransformer3DModel), Model(name='WAN 2.1 14B T2V', url='https://huggingface.co/Wan-AI/Wan2.1-T2V-14B-Diffusers', repo='Wan-AI/Wan2.1-T2V-14B-Diffusers', repo_cls=diffusers.WanPipeline, - te_cls=transformers.T5EncoderModel, + te_cls=transformers.UMT5EncoderModel, dit_cls=diffusers.WanTransformer3DModel), Model(name='WAN 2.1 14B I2V 480p', url='https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-480P-Diffusers', repo='Wan-AI/Wan2.1-I2V-14B-480P-Diffusers', repo_cls=diffusers.WanImageToVideoPipeline, - te_cls=transformers.T5EncoderModel, + te_cls=transformers.UMT5EncoderModel, dit_cls=diffusers.WanTransformer3DModel), Model(name='WAN 2.1 14B I2V 720p', url='https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-720P-Diffusers', repo='Wan-AI/Wan2.1-I2V-14B-720P-Diffusers', repo_cls=diffusers.WanImageToVideoPipeline, - te_cls=transformers.T5EncoderModel, + te_cls=transformers.UMT5EncoderModel, dit_cls=diffusers.WanTransformer3DModel), Model(name='WAN 2.1 14B FLF2V 720p', url='https://huggingface.co/Wan-AI/Wan2.1-FLF2V-14B-720P', repo='Wan-AI/Wan2.1-FLF2V-14B-720P-diffusers', repo_cls=diffusers.WanImageToVideoPipeline, - te_cls=transformers.T5EncoderModel, + te_cls=transformers.UMT5EncoderModel, dit_cls=diffusers.WanTransformer3DModel), Model(name='WAN 2.1 VACE 1.3B', url='https://huggingface.co/Wan-AI/Wan2.1-VACE-1.3B-diffusers', repo='Wan-AI/Wan2.1-VACE-1.3B-diffusers', repo_cls=diffusers.WanVACEPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel), + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanVACETransformer3DModel), Model(name='WAN 2.1 VACE 14B', url='https://huggingface.co/Wan-AI/Wan2.1-VACE-14B-diffusers', repo='Wan-AI/Wan2.1-VACE-14B-diffusers', repo_cls=diffusers.WanVACEPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel), + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanVACETransformer3DModel), ], 'SkyReels V2': [ Model(name='None'), diff --git a/modules/video_models/video_load.py b/modules/video_models/video_load.py index 72c411166..d41e3d463 100644 --- a/modules/video_models/video_load.py +++ b/modules/video_models/video_load.py @@ -8,7 +8,7 @@ loaded_model = None def load_model(selected: models_def.Model): - if selected is None: + if selected is None or selected.te_cls is None or selected.dit_cls is None: return '' global loaded_model # pylint: disable=global-statement if loaded_model == selected.name: @@ -25,6 +25,21 @@ def load_model(selected: models_def.Model): # text encoder try: load_args, quant_args = model_quant.get_dit_args({}, module='TE', device_map=True) + + # loader deduplication of text-encoder models + if selected.te_cls.__name__ == 'T5EncoderModel' and shared.opts.te_shared_t5: + selected.te = 'Disty0/t5-xxl' + selected.te_folder = '' + selected.te_revision = None + if selected.te_cls.__name__ == 'UMT5EncoderModel' and shared.opts.te_shared_t5: + selected.te = 'Wan-AI/Wan2.2-TI2V-5B-Diffusers' + selected.te_folder = 'text_encoder' + selected.te_revision = None + if selected.te_cls.__name__ == 'LlamaModel' and shared.opts.te_shared_t5: + selected.te = 'hunyuanvideo-community/HunyuanVideo' + selected.te_folder = 'text_encoder' + selected.te_revision = None + shared.log.debug(f'Video load: module=te repo="{selected.te or selected.repo}" folder="{selected.te_folder}" cls={selected.te_cls.__name__} quant={model_quant.get_quant_type(quant_args)}') kwargs["text_encoder"] = selected.te_cls.from_pretrained( pretrained_model_name_or_path=selected.te or selected.repo, @@ -40,12 +55,11 @@ def load_model(selected: models_def.Model): # transformer try: - if isinstance(selected.dit_folder, list) or isinstance(selected.dit_folder, tuple): - # wan a14b has transformer and transformer_2 - for dit_folder in selected.dit_folder: + def load_dit_folder(dit_folder): + if dit_folder is not None and dit_folder not in kwargs: # get a new quant arg on every loop to prevent the quant config classes getting entangled load_args, quant_args = model_quant.get_dit_args({}, module='Model', device_map=True) - shared.log.debug(f'Video load: module=transformer repo="{selected.dit or selected.repo}" folder="{dit_folder}" cls={selected.dit_cls.__name__} quant={model_quant.get_quant_type(quant_args)}') + shared.log.debug(f'Video load: module=transformer repo="{selected.dit or selected.repo}" module="{dit_folder}" folder="{dit_folder}" cls={selected.dit_cls.__name__} quant={model_quant.get_quant_type(quant_args)}') kwargs[dit_folder] = selected.dit_cls.from_pretrained( pretrained_model_name_or_path=selected.dit or selected.repo, subfolder=dit_folder, @@ -54,17 +68,16 @@ def load_model(selected: models_def.Model): **load_args, **quant_args ) + else: + shared.log.debug(f'Video load: module=transformer repo="{selected.dit or selected.repo}" module="{dit_folder}" folder="{dit_folder}" cls={selected.dit_cls.__name__} skip') + + if selected.dit_folder is None: + selected.dit_folder = ['transformer'] + if isinstance(selected.dit_folder, list) or isinstance(selected.dit_folder, tuple): + for dit_folder in selected.dit_folder: # wan a14b has transformer and transformer_2 + load_dit_folder(dit_folder) else: - load_args, quant_args = model_quant.get_dit_args({}, module='Model', device_map=True) - shared.log.debug(f'Video load: module=transformer repo="{selected.dit or selected.repo}" folder="{selected.dit_folder}" cls={selected.dit_cls.__name__} quant={model_quant.get_quant_type(quant_args)}') - kwargs["transformer"] = selected.dit_cls.from_pretrained( - pretrained_model_name_or_path=selected.dit or selected.repo, - subfolder=selected.dit_folder, - revision=selected.dit_revision or selected.repo_revision, - cache_dir=shared.opts.hfcache_dir, - **load_args, - **quant_args - ) + load_dit_folder(selected.dit_folder) except Exception as e: shared.log.error(f'video load: module=transformer cls={selected.dit_cls.__name__} {e}') errors.display(e, 'video') @@ -91,17 +104,23 @@ def load_model(selected: models_def.Model): shared.sd_model.sd_model_hash = None sd_models.set_diffuser_options(shared.sd_model, offload=False) + decode, text, image, slicing, tiling = False, False, False, False, False if selected.vae_hijack and hasattr(shared.sd_model.vae, 'decode'): sd_hijack_vae.init_hijack(shared.sd_model) + decode = True if selected.te_hijack and hasattr(shared.sd_model, 'encode_prompt'): sd_hijack_te.init_hijack(shared.sd_model) + text = True if selected.image_hijack and hasattr(shared.sd_model, 'encode_image'): shared.sd_model.orig_encode_image = shared.sd_model.encode_image shared.sd_model.encode_image = video_utils.hijack_encode_image - if hasattr(shared.sd_model.vae, 'enable_slicing'): + image = True + if hasattr(shared.sd_model, 'vae') and hasattr(shared.sd_model.vae, 'enable_slicing'): shared.sd_model.vae.enable_slicing() - if hasattr(shared.sd_model.vae, 'enable_tiling'): + slicing = True + if hasattr(shared.sd_model, 'vae') and hasattr(shared.sd_model.vae, 'enable_tiling'): shared.sd_model.vae.enable_tiling() + tiling = True if hasattr(shared.sd_model, "set_progress_bar_config"): shared.sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m', ncols=80, colour='#327fba') @@ -111,5 +130,6 @@ def load_model(selected: models_def.Model): loaded_model = selected.name msg = f'Video load: cls={shared.sd_model.__class__.__name__} model="{selected.name}" time={t1-t0:.2f}' shared.log.info(msg) + shared.log.debug(f'Video hijacks: decode={decode} text={text} image={image} slicing={slicing} tiling={tiling}') shared.state.end(jobid) return msg diff --git a/modules/video_models/video_overrides.py b/modules/video_models/video_overrides.py index 5ce02e20a..856da62e6 100644 --- a/modules/video_models/video_overrides.py +++ b/modules/video_models/video_overrides.py @@ -19,6 +19,15 @@ def load_override(selected: Model): # WAN if 'WAN 2.1 14B' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLWan.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir) + if 'A14B' in selected.name or '14B VACE' in selected.name: + if shared.opts.model_wan_stage == 'combined': + kwargs['boundary_ratio'] = shared.opts.model_wan_boundary + elif shared.opts.model_wan_stage == 'high noise': + kwargs['transformer_2'] = None + kwargs['boundary_ratio'] = 0.0 + elif shared.opts.model_wan_stage == 'low noise': + kwargs['boundary_ratio'] = 1.0 + kwargs['transformer'] = None debug(f'Video overrides: model="{selected.name}" kwargs={list(kwargs)}') return kwargs @@ -32,19 +41,23 @@ def set_overrides(p: processing.StableDiffusionProcessingVideo, selected: Model) if selected.name == 'Latte 1 T2V': p.task_args['enable_temporal_attentions'] = True p.task_args['video_length'] = 16 * (max(p.frames // 16, 1)) + # SkyReels + if 'SkyReelsV2DiffusionForcing' in cls: + p.task_args['overlap_history'] = 17 # LTX if cls == 'LTXImageToVideoPipeline' or cls == 'LTXConditionPipeline': p.task_args['generator'] = None if cls == 'LTXConditionPipeline': p.task_args['strength'] = p.denoising_strength + if 'LTX' in cls: + p.task_args['width'] = 32 * (p.width // 32) + p.task_args['height'] = 32 * (p.height // 32) # WAN if 'Wan' in cls: p.task_args['width'] = 16 * (p.width // 16) p.task_args['height'] = 16 * (p.height // 16) p.frames = 4 * (max(p.frames // 4, 1)) + 1 - # LTX - if 'LTX' in cls: - p.task_args['width'] = 32 * (p.width // 32) - p.task_args['height'] = 32 * (p.height // 32) - if 'SkyReelsV2DiffusionForcing' in cls: - p.task_args['overlap_history'] = 17 + # WAN VACE + if 'WanVACEPipeline' in cls: + if (getattr(p, 'init_images', None) is not None) and (len(p.init_images) > 0): + p.task_args['reference_images'] = p.init_images diff --git a/modules/video_models/video_prompt.py b/modules/video_models/video_prompt.py new file mode 100644 index 000000000..bbb29c409 --- /dev/null +++ b/modules/video_models/video_prompt.py @@ -0,0 +1,21 @@ +from modules import shared, extra_networks, ui_video_vlm + + +def prepare_prompt(p, init_image, prompt:str, vlm_enhance:bool, vlm_model:str, vlm_system_prompt:str): + p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) + p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) + shared.prompt_styles.apply_styles_to_extra(p) + p.prompts, p.network_data = extra_networks.parse_prompts([p.prompt]) + extra_networks.activate(p) + prompt = p.prompts[0] + + new_prompt = ui_video_vlm.enhance_prompt( + enable=vlm_enhance, + model=vlm_model, + image=init_image, + prompt=prompt, + system_prompt=vlm_system_prompt, + ) + if new_prompt is not None and len(new_prompt) > 0: + prompt = new_prompt + return prompt diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 508980bad..40f7e154f 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -1,16 +1,17 @@ import os import time from modules import shared, errors, sd_models, processing, devices, images, ui_common -from modules.video_models import models_def, video_utils, video_load, video_vae, video_overrides +from modules.video_models import models_def, video_utils, video_load, video_vae, video_overrides, video_save, video_prompt debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None def generate(*args, **kwargs): - task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, init_strength, last_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args + task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, init_strength, last_image, vae_type, vae_tile_frames, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, vlm_enhance, vlm_model, vlm_system_prompt, override_settings = args if engine is None or model is None or engine == 'None' or model == 'None': return video_utils.queue_err('model not selected') + # videojob = shared.state.begin('Video') found = [model.name for model in models_def.models.get(engine, [])] selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None if not shared.sd_loaded: @@ -51,9 +52,12 @@ def generate(*args, **kwargs): p.script_args = None p.state = ui_state p.do_not_save_grid = True - p.do_not_save_samples = not save_frames + p.do_not_save_samples = not mp4_frames p.outpath_samples = shared.opts.outdir_samples or shared.opts.outdir_video - if 'I2V' in model: + if 'T2V' in model: + if init_image is not None: + shared.log.warning('Video: op=T2V init image not supported') + elif 'I2V' in model: if init_image is None: return video_utils.queue_err('init image not set') p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') @@ -66,9 +70,10 @@ def generate(*args, **kwargs): p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') p.task_args['last_image'] = images.resize_image(resize_mode=2, im=last_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') shared.log.debug(f'Video: op=FLF2V init={init_image} last={last_image} resized={p.task_args["image"]}') - elif 'T2V' in model: + elif 'VACE' in model: if init_image is not None: - shared.log.warning('Video: op=T2V init image not supported') + p.task_args['reference_images'] = [images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')] + shared.log.debug(f'Video: op=VACE reference={init_image} resized={p.task_args["reference_images"]}') else: shared.log.warning(f'Video: unknown model type "{model}"') @@ -76,6 +81,8 @@ def generate(*args, **kwargs): shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) devices.torch_gc(force=True, reason='video') + prompt = video_prompt.prepare_prompt(p, init_image, prompt, vlm_enhance, vlm_model, vlm_system_prompt) + # set args processing.fix_seed(p) video_vae.set_vae_params(p) @@ -115,6 +122,24 @@ def generate(*args, **kwargs): if processed is None or len(processed.images) == 0: return video_utils.queue_err('processing failed') shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') - video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) + + # video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) # legacy video save from list of images + pixels = video_save.images_to_tensor(processed.images) + _num_frames, video_file = video_save.save_video( + pixels=pixels, + mp4_fps=mp4_fps, + mp4_codec=mp4_codec, + mp4_opt=mp4_opt, + mp4_ext=mp4_ext, + mp4_sf=mp4_sf, + mp4_video=mp4_video, + mp4_frames=mp4_frames, + mp4_interpolate=mp4_interpolate, + metadata={}, + ) + if not mp4_frames: + processed.images = [] + generation_info_js = processed.js() if processed is not None else '' + # shared.state.end(videojob) return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) diff --git a/modules/video_models/video_save.py b/modules/video_models/video_save.py index 2ce71f0af..24775c5a1 100644 --- a/modules/video_models/video_save.py +++ b/modules/video_models/video_save.py @@ -2,9 +2,11 @@ import os import time import datetime import cv2 +import numpy as np import torch import einops from modules import shared, errors ,timer, rife +from modules.video_models.video_utils import check_av def get_video_filename(frames:int, codec:str): @@ -13,14 +15,25 @@ def get_video_filename(frames:int, codec:str): return output_filename +def images_to_tensor(images): + if images is None or len(images) == 0: + return None + array = [torch.from_numpy(np.array(image)) for image in images] + tensor = torch.stack(array, dim=0) # n h w c + tensor = tensor.unsqueeze(0) # 1, n, h, w, c + tensor = tensor.permute(0, 4, 1, 2, 3).contiguous() # 1, c, n, h, w + tensor = (tensor.float() / 127.5) - 1.0 # from [0,255] to [-1,1] + # shared.log.debug(f'Video output: images={len(images)} tensor={tensor.shape}') + return tensor + + def atomic_save_video(filename, tensor:torch.Tensor, fps:float=24, codec:str='libx264', pix_fmt:str='yuv420p', options:str='', metadata:dict={}, pbar=None): - try: - import av - av.logging.set_level(av.logging.ERROR) # pylint: disable=c-extension-no-member - except Exception as e: - shared.log.error(f'Video: {e}') + av = check_av() + if av is None or av is False: + shared.log.error('Video: ffmpeg/av not available') return + savejob = shared.state.begin('Save video') frames, height, width, _channels = tensor.shape rate = round(fps) options_str = options @@ -54,6 +67,7 @@ def atomic_save_video(filename, tensor:torch.Tensor, fps:float=24, codec:str='li for packet in stream.encode(): # flush container.mux(packet) shared.state.outputs(filename) + shared.state.end(savejob) def save_video( @@ -73,13 +87,16 @@ def save_video( output_video = None if pixels is None: return 0, output_video + if not torch.is_tensor(pixels): + shared.log.error(f'Video: type={type(pixels)} not a tensor') + return 0, output_video t_save = time.time() n, _c, t, h, w = pixels.shape size = pixels.element_size() * pixels.numel() shared.log.debug(f'Video: video={mp4_video} export={mp4_frames} safetensors={mp4_sf} interpolate={mp4_interpolate}') shared.log.debug(f'Video: encode={t} raw={size} latent={pixels.shape} fps={mp4_fps} codec={mp4_codec} ext={mp4_ext} options="{mp4_opt}"') - jobid = shared.state.begin('Save video') try: + preparejob = shared.state.begin('Prepare video') if stream is not None: stream.output_queue.push(('progress', (None, 'Saving video...'))) if mp4_interpolate > 0: @@ -111,6 +128,8 @@ def save_video( shared.state.outputs(fn) cv2.imwrite(fn, image) + shared.state.end(preparejob) + if mp4_video and (mp4_codec != 'none'): output_video = f'{output_filename}.{mp4_ext}' atomic_save_video(output_video, tensor=x, fps=mp4_fps, codec=mp4_codec, options=mp4_opt, metadata=metadata, pbar=pbar) @@ -125,5 +144,4 @@ def save_video( shared.log.error(f'Video save: raw={size} {e}') errors.display(e, 'video') timer.process.add('save', time.time()-t_save) - shared.state.end(jobid) return t, output_video diff --git a/modules/video_models/video_ui.py b/modules/video_models/video_ui.py index 5b103d023..cd78f45a2 100644 --- a/modules/video_models/video_ui.py +++ b/modules/video_models/video_ui.py @@ -1,6 +1,6 @@ import os import gradio as gr -from modules import shared, sd_models, ui_common, ui_sections, ui_symbols, call_queue +from modules import shared, sd_models, ui_common, ui_sections, ui_symbols, ui_video_vlm, call_queue from modules.ui_components import ToolButton from modules.video_models import models_def, video_utils from modules.video_models import video_run @@ -112,19 +112,31 @@ def create_ui(prompt, negative, styles, overrides): init_image = gr.Image(elem_id="video_image", show_label=False, type="pil", image_mode="RGB", width=256, height=256) gr.HTML("
  Last image") last_image = gr.Image(elem_id="video_last", show_label=False, type="pil", image_mode="RGB", width=256, height=256) - with gr.Accordion(open=False, label="Output", elem_id='video_output_accordion'): + + vlm_enhance, vlm_model, vlm_system_prompt = ui_video_vlm.create_ui(prompt_element=prompt, image_element=init_image) + + with gr.Accordion(label="Video", open=False, elem_id='video_output_accordion'): with gr.Row(): - save_frames = gr.Checkbox(label='Save image frames', value=False, elem_id="video_save_frames") + mp4_fps = gr.Slider(label="FPS", minimum=1, maximum=60, value=24, step=1) + mp4_interpolate = gr.Slider(label="Video interpolation", minimum=0, maximum=10, value=0, step=1) with gr.Row(): - video_type, video_duration, video_loop, video_pad, video_interpolate = ui_sections.create_video_inputs(tab='video', show_always=True) + mp4_codec = gr.Dropdown(label="Video codec", choices=['none', 'libx264'], value='libx264', type='value') + ui_common.create_refresh_button(mp4_codec, video_utils.get_codecs, elem_id="framepack_mp4_codec_refresh") + mp4_ext = gr.Textbox(label="Video format", value='mp4', elem_id="framepack_mp4_ext") + mp4_opt = gr.Textbox(label="Video options", value='crf:16', elem_id="framepack_mp4_ext") + with gr.Row(): + mp4_video = gr.Checkbox(label='Video save video', value=True, elem_id="framepack_mp4_video") + mp4_frames = gr.Checkbox(label='Video save frames', value=False, elem_id="framepack_mp4_frames") + mp4_sf = gr.Checkbox(label='Video save safetensors', value=False, elem_id="framepack_mp4_sf") + # output panel with gallery and video tabs with gr.Column(elem_id='video-output-column', scale=2) as _column_output: with gr.Tabs(elem_classes=['video-output-tabs'], elem_id='video-output-tabs'): - with gr.Tab('Frames', id='out-gallery'): - gallery, gen_info, html_info, _html_info_formatted, html_log = ui_common.create_output_panel("video", prompt=prompt, preview=False, transfer=False, scale=2) with gr.Tab('Video', id='out-video'): video = gr.Video(label="Output", show_label=False, elem_id='control_output_video', elem_classes=['control-image'], height=512, autoplay=False) + with gr.Tab('Frames', id='out-gallery'): + gallery, gen_info, html_info, _html_info_formatted, html_log = ui_common.create_output_panel("video", prompt=prompt, preview=False, transfer=False, scale=2) # connect reuse seed button ui_common.connect_reuse_seed(seed, reuse_seed, gen_info, is_subseed=False) @@ -150,8 +162,8 @@ def create_ui(prompt, negative, styles, overrides): guidance_scale, guidance_true, init_image, init_strength, last_image, vae_type, vae_tile_frames, - save_frames, - video_type, video_duration, video_loop, video_pad, video_interpolate, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, + vlm_enhance, vlm_model, vlm_system_prompt, overrides, ] video_outputs = [ diff --git a/modules/video_models/video_utils.py b/modules/video_models/video_utils.py index 47faeac13..f29c43906 100644 --- a/modules/video_models/video_utils.py +++ b/modules/video_models/video_utils.py @@ -2,6 +2,7 @@ import os import sys import time from PIL import Image +from installer import install from modules import shared, sd_models, timer, errors, devices @@ -18,8 +19,10 @@ def get_url(url): def check_av(): + install('av') try: import av + av.logging.set_level(av.logging.ERROR) # pylint: disable=c-extension-no-member except Exception as e: shared.log.error(f'av package: {e}') return False diff --git a/modules/windows_hip_ffi.py b/modules/windows_hip_ffi.py new file mode 100644 index 000000000..c612aa43d --- /dev/null +++ b/modules/windows_hip_ffi.py @@ -0,0 +1,47 @@ +import sys + +if sys.platform == "win32": + import os + import ctypes + import ctypes.wintypes + + class hipDeviceProp(ctypes.Structure): + _fields_ = [ + ('__front__', ctypes.c_byte * 396), + ('gcnArchName', ctypes.c_char * 256), + ('__rear__', ctypes.c_byte * 820) + ] + + class HIP: + def __init__(self): + ctypes.windll.kernel32.LoadLibraryA.restype = ctypes.wintypes.HMODULE + ctypes.windll.kernel32.LoadLibraryA.argtypes = [ctypes.c_char_p] + path = os.environ.get("windir", "C:\\Windows") + "\\System32\\amdhip64_6.dll" + if not os.path.isfile(path): + path = os.environ.get("windir", "C:\\Windows") + "\\System32\\amdhip64_7.dll" + assert os.path.isfile(path) + self.handle = ctypes.windll.kernel32.LoadLibraryA(path.encode('utf-8')) + ctypes.windll.kernel32.GetLastError.restype = ctypes.wintypes.DWORD + ctypes.windll.kernel32.GetLastError.argtypes = [] + assert ctypes.windll.kernel32.GetLastError() == 0 + ctypes.windll.kernel32.GetProcAddress.restype = ctypes.c_void_p + ctypes.windll.kernel32.GetProcAddress.argtypes = [ctypes.wintypes.HMODULE, ctypes.c_char_p] + self.hipGetDeviceCount = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(ctypes.c_int))( + ctypes.windll.kernel32.GetProcAddress(self.handle, b"hipGetDeviceCount")) + self.hipGetDeviceProperties = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(hipDeviceProp), ctypes.c_int)( + ctypes.windll.kernel32.GetProcAddress(self.handle, b"hipGetDeviceProperties")) + + def __del__(self): + # Hopefully this will prevent conflicts with amdhip64_7.dll from ROCm Python packages or HIP SDK + ctypes.windll.kernel32.FreeLibrary.argtypes = [ctypes.wintypes.HMODULE] + ctypes.windll.kernel32.FreeLibrary(self.handle) + + def get_device_count(self): + count = ctypes.c_int() + assert self.hipGetDeviceCount(ctypes.byref(count)) == 0 + return count.value + + def get_device_properties(self, device_id): + prop = hipDeviceProp() + assert self.hipGetDeviceProperties(ctypes.byref(prop), device_id) == 0 + return prop diff --git a/modules/zluda_installer.py b/modules/zluda_installer.py index da0ee290f..c6fb55c23 100644 --- a/modules/zluda_installer.py +++ b/modules/zluda_installer.py @@ -119,12 +119,13 @@ def link_or_copy(src: os.PathLike, dst: os.PathLike): def load(): + assert isinstance(rocm.environment, rocm.ROCmEnvironment) global core, ml, hipBLASLt_enabled, MIOpen_enabled # pylint: disable=global-statement core = Core(ctypes.windll.LoadLibrary(os.path.join(path, 'nvcuda.dll'))) ml = ZLUDALibrary(ctypes.windll.LoadLibrary(os.path.join(path, 'nvml.dll'))) is_nightly = core.get_nightly_flag() == 1 - hipBLASLt_enabled = is_nightly and os.path.exists(rocm.blaslt_tensile_libpath) and os.path.exists(os.path.join(rocm.path, "bin", "hipblaslt.dll")) and default_agent is not None - MIOpen_enabled = is_nightly and os.path.exists(os.path.join(rocm.path, "bin", "MIOpen.dll")) + hipBLASLt_enabled = is_nightly and os.path.exists(rocm.blaslt_tensile_libpath) and os.path.exists(os.path.join(rocm.environment.path, "bin", "hipblaslt.dll")) and default_agent is not None + MIOpen_enabled = is_nightly and os.path.exists(os.path.join(rocm.environment.path, "bin", "MIOpen.dll")) if hipBLASLt_enabled: if not default_agent.blaslt_supported: @@ -147,22 +148,22 @@ def load(): os.environ["ZLUDA_NVRTC_LIB"] = os.path.join([v for v in site.getsitepackages() if v.endswith("site-packages")][0], "torch", "lib", "nvrtc64_112_0.dll") for v in HIPSDK_TARGETS: - ctypes.windll.LoadLibrary(os.path.join(rocm.path, 'bin', v)) + ctypes.windll.LoadLibrary(os.path.join(rocm.environment.path, 'bin', v)) for v in DLL_MAPPING.values(): ctypes.windll.LoadLibrary(os.path.join(path, v)) if hipBLASLt_enabled: os.environ.setdefault("DISABLE_ADDMM_CUDA_LT", "0") - ctypes.windll.LoadLibrary(os.path.join(rocm.path, 'bin', 'hipblaslt.dll')) + ctypes.windll.LoadLibrary(os.path.join(rocm.environment.path, 'bin', 'hipblaslt.dll')) ctypes.windll.LoadLibrary(os.path.join(path, 'cublasLt64_11.dll')) else: os.environ["DISABLE_ADDMM_CUDA_LT"] = "1" if MIOpen_enabled: - ctypes.windll.LoadLibrary(os.path.join(rocm.path, 'bin', 'MIOpen.dll')) + ctypes.windll.LoadLibrary(os.path.join(rocm.environment.path, 'bin', 'MIOpen.dll')) ctypes.windll.LoadLibrary(os.path.join(path, 'cudnn64_9.dll')) - def conceal(): + def postinstall(): import torch torch.version.hip = rocm.version platform = sys.platform @@ -175,4 +176,4 @@ def load(): def _join_rocm_home(*paths) -> str: return os.path.join(cpp_extension.ROCM_HOME, *paths) cpp_extension._join_rocm_home = _join_rocm_home # pylint: disable=protected-access - rocm.conceal = conceal + rocm.postinstall = postinstall diff --git a/package.json b/package.json index 1b0b95bca..e42d5a2fd 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "venv": ". venv/bin/activate", "start": ". venv/bin/activate; python launch.py --debug", "localize": "node cli/localize.js", - "packages": ". venv/bin/activate && pip install --upgrade transformers accelerate huggingface_hub safetensors tokenizers peft compel pytorch_lightning pylint ruff", + "packages": ". venv/bin/activate && pip install --upgrade transformers accelerate huggingface_hub safetensors tokenizers peft pytorch_lightning pylint ruff", "eslint": "eslint . javascript/ extensions-builtin/sdnext-modernui/javascript/", "ruff": ". venv/bin/activate && ruff check", "pylint": ". venv/bin/activate && pylint *.py modules/ pipelines/ scripts/ extensions-builtin/ | grep -v '^*'", diff --git a/pipelines/flux/flux_legacy_loader.py b/pipelines/flux/flux_legacy_loader.py index 6b6f9d294..cb3dad3c1 100644 --- a/pipelines/flux/flux_legacy_loader.py +++ b/pipelines/flux/flux_legacy_loader.py @@ -130,7 +130,7 @@ def load_quants(kwargs, repo_id, cache_dir, allow_quant): # pylint: disable=unus shared.log.error(f'Load module: quant=Nunchaku module=transformer repo="{repo_id}" unsupported') if nunchaku_repo is not None: shared.log.debug(f'Load module: quant=Nunchaku module=transformer repo="{nunchaku_repo}" precision={nunchaku_precision} offload={shared.opts.nunchaku_offload} attention={shared.opts.nunchaku_attention}') - kwargs['transformer'] = nunchaku.NunchakuFluxTransformer2dModel.from_pretrained(nunchaku_repo, offload=shared.opts.nunchaku_offload, torch_dtype=devices.dtype) + kwargs['transformer'] = nunchaku.NunchakuFluxTransformer2dModel.from_pretrained(nunchaku_repo, offload=shared.opts.nunchaku_offload, torch_dtype=devices.dtype, cache_dir=cache_dir) kwargs['transformer'].quantization_method = 'SVDQuant' if shared.opts.nunchaku_attention: kwargs['transformer'].set_attention_impl("nunchaku-fp16") @@ -142,7 +142,7 @@ def load_quants(kwargs, repo_id, cache_dir, allow_quant): # pylint: disable=unus nunchaku_precision = nunchaku.utils.get_precision() nunchaku_repo = 'mit-han-lab/nunchaku-t5/awq-int4-flux.1-t5xxl.safetensors' shared.log.debug(f'Load module: quant=Nunchaku module=t5 repo="{nunchaku_repo}" precision={nunchaku_precision}') - kwargs['text_encoder_2'] = nunchaku.NunchakuT5EncoderModel.from_pretrained(nunchaku_repo, torch_dtype=devices.dtype) + kwargs['text_encoder_2'] = nunchaku.NunchakuT5EncoderModel.from_pretrained(nunchaku_repo, torch_dtype=devices.dtype, cache_dir=cache_dir) kwargs['text_encoder_2'].quantization_method = 'SVDQuant' if 'text_encoder_2' not in kwargs and model_quant.check_quant('TE'): load_args, quant_args = model_quant.get_dit_args(diffusers_load_config, module='TE', device_map=True) diff --git a/pipelines/flux/flux_nunchaku.py b/pipelines/flux/flux_nunchaku.py index 1aba177aa..9a737d103 100644 --- a/pipelines/flux/flux_nunchaku.py +++ b/pipelines/flux/flux_nunchaku.py @@ -6,7 +6,9 @@ def load_flux_nunchaku(repo_id): nunchaku_precision = nunchaku.utils.get_precision() nunchaku_repo = None transformer = None - if 'flux.1-dev' in repo_id.lower(): + if 'srpo' in repo_id.lower(): + pass + elif 'flux.1-dev' in repo_id.lower(): nunchaku_repo = f"nunchaku-tech/nunchaku-flux.1-dev/svdq-{nunchaku_precision}_r32-flux.1-dev.safetensors" elif 'flux.1-schnell' in repo_id.lower(): nunchaku_repo = f"nunchaku-tech/nunchaku-flux.1-schnell/svdq-{nunchaku_precision}_r32-flux.1-schnell.safetensors" @@ -24,7 +26,12 @@ def load_flux_nunchaku(repo_id): shared.log.error(f'Load module: quant=Nunchaku module=transformer repo="{repo_id}" unsupported') if nunchaku_repo is not None: shared.log.debug(f'Load module: quant=Nunchaku module=transformer repo="{nunchaku_repo}" precision={nunchaku_precision} offload={shared.opts.nunchaku_offload} attention={shared.opts.nunchaku_attention}') - transformer = nunchaku.NunchakuFluxTransformer2dModel.from_pretrained(nunchaku_repo, offload=shared.opts.nunchaku_offload, torch_dtype=devices.dtype) + transformer = nunchaku.NunchakuFluxTransformer2dModel.from_pretrained( + nunchaku_repo, + offload=shared.opts.nunchaku_offload, + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + ) transformer.quantization_method = 'SVDQuant' if shared.opts.nunchaku_attention: transformer.set_attention_impl("nunchaku-fp16") diff --git a/pipelines/generic.py b/pipelines/generic.py index 2c5f0b847..f14f88ab8 100644 --- a/pipelines/generic.py +++ b/pipelines/generic.py @@ -48,7 +48,7 @@ def load_transformer(repo_id, cls_name, load_config={}, subfolder="transformer", **quant_args, ) else: - shared.log.debug(f'Load model: transformer="{repo_id}" cls={cls_name.__name__} quant="{quant_type}" args={load_args}') + shared.log.debug(f'Load model: transformer="{repo_id}" cls={cls_name.__name__} subfolder={subfolder} quant="{quant_type}" args={load_args}') if dtype is not None: load_args['torch_dtype'] = dtype if subfolder is not None: @@ -61,9 +61,13 @@ def load_transformer(repo_id, cls_name, load_config={}, subfolder="transformer", **load_args, **quant_args, ) + sd_models.allow_post_quant = False # we already handled it if shared.opts.diffusers_offload_mode != 'none' and transformer is not None: sd_models.move_model(transformer, devices.cpu) + + if (transformer is not None) and (quant_type is not None) and (quant_args.get('quantization_config', None) is not None): # attach quantization_config + transformer.quantization_config = quant_args.get('quantization_config', None) except Exception as e: shared.log.error(f'Load model: transformer="{repo_id}" cls={cls_name.__name__} {e}') errors.display(e, 'Load:') @@ -157,6 +161,9 @@ def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder sd_models.allow_post_quant = False # we already handled it if shared.opts.diffusers_offload_mode != 'none' and text_encoder is not None: sd_models.move_model(text_encoder, devices.cpu) + + if (text_encoder is not None) and (quant_type is not None) and (quant_args.get('quantization_config', None) is not None): # attach quantization_config + text_encoder.quantization_config = quant_args.get('quantization_config', None) except Exception as e: shared.log.error(f'Load model: text_encoder="{repo_id}" cls={cls_name.__name__} {e}') errors.display(e, 'Load:') diff --git a/pipelines/model_hidream.py b/pipelines/model_hidream.py index 7f49997fc..120a89479 100644 --- a/pipelines/model_hidream.py +++ b/pipelines/model_hidream.py @@ -52,6 +52,10 @@ def load_hidream(checkpoint_info, diffusers_load_config={}): diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["hidream-e1"] = diffusers.HiDreamImagePipeline diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["hidream-e1"] = HiDreamImageEditingPipeline diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["hidream-e1"] = HiDreamImageEditingPipeline + if transformer and 'E1-1' in repo_id: + transformer.max_seq = 8192 + elif transformer and 'E1' in repo_id: + transformer.max_seq = 4608 else: shared.log.error(f'Load model: type=HiDream model="{checkpoint_info.name}" repo="{repo_id}" not recognized') return False diff --git a/pipelines/model_qwen.py b/pipelines/model_qwen.py index bb65df53e..08faf5624 100644 --- a/pipelines/model_qwen.py +++ b/pipelines/model_qwen.py @@ -1,18 +1,24 @@ import transformers import diffusers from modules import shared, devices, sd_models, model_quant, sd_hijack_te, sd_hijack_vae -from pipelines import generic def load_qwen(checkpoint_info, diffusers_load_config={}): + from pipelines import generic, qwen repo_id = sd_models.path_to_repo(checkpoint_info) + repo_subfolder = checkpoint_info.subfolder sd_models.hf_auth_check(checkpoint_info) transformer = None load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, module='Model') shared.log.debug(f'Load model: type=Qwen model="{checkpoint_info.name}" repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') - if 'Edit' in repo_id: + if '2509' in repo_id : + cls_name = diffusers.QwenImageEditPlusPipeline + diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["qwen-image"] = diffusers.QwenImageEditPlusPipeline + diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["qwen-image"] = diffusers.QwenImageEditPlusPipeline + diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["qwen-image"] = diffusers.QwenImageEditPlusPipeline + elif 'Edit' in repo_id: cls_name = diffusers.QwenImageEditPipeline diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["qwen-image"] = diffusers.QwenImageEditPipeline diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["qwen-image"] = diffusers.QwenImageEditPipeline @@ -24,10 +30,7 @@ def load_qwen(checkpoint_info, diffusers_load_config={}): diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["qwen-image"] = diffusers.QwenImageInpaintPipeline if model_quant.check_nunchaku('Model'): - from pipelines.qwen.qwen_nunchaku import load_qwen_nunchaku - transformer = load_qwen_nunchaku(repo_id) - # if transformer is not None: - # cls_name = nunchaku.pipeline.pipeline_qwenimage.NunchakuQwenImagePipeline # we dont need this + transformer = qwen.load_qwen_nunchaku(repo_id) if 'Qwen-Image-Distill-Full' in repo_id: repo_transformer = repo_id @@ -35,19 +38,29 @@ def load_qwen(checkpoint_info, diffusers_load_config={}): repo_id = 'Qwen/Qwen-Image' else: repo_transformer = repo_id - transformer_subfolder = "transformer" + if repo_subfolder is not None: + transformer_subfolder = repo_subfolder + '/transformer' + else: + transformer_subfolder = "transformer" if transformer is None: - transformer = generic.load_transformer(repo_transformer, subfolder=transformer_subfolder, cls_name=diffusers.QwenImageTransformer2DModel, load_config=diffusers_load_config, modules_dtype_dict={"minimum_6bit": ["pos_embed", "time_text_embed", "img_in", "txt_in", "norm_out", "transformer_blocks.0.img_mod.1.weight"]}) + transformer = generic.load_transformer( + repo_transformer, + subfolder=transformer_subfolder, + cls_name=diffusers.QwenImageTransformer2DModel, + load_config=diffusers_load_config, + modules_to_not_convert=["transformer_blocks.0.img_mod.1.weight"], + ) repo_te = 'Qwen/Qwen-Image' text_encoder = generic.load_text_encoder(repo_te, cls_name=transformers.Qwen2_5_VLForConditionalGeneration, load_config=diffusers_load_config) - # NunchakuQwenImagePipeline + repo_id, repo_subfolder = qwen.check_qwen_pruning(repo_id, repo_subfolder) pipe = cls_name.from_pretrained( repo_id, transformer=transformer, text_encoder=text_encoder, + subfolder=repo_subfolder, cache_dir=shared.opts.diffusers_dir, **load_args, ) diff --git a/pipelines/model_sana.py b/pipelines/model_sana.py index d4ec574f6..9584263a3 100644 --- a/pipelines/model_sana.py +++ b/pipelines/model_sana.py @@ -10,9 +10,8 @@ def load_quants(kwargs, repo_id, cache_dir): import nunchaku nunchaku_precision = nunchaku.utils.get_precision() nunchaku_repo = "nunchaku-tech/nunchaku-sana/svdq-int4_r32-sana1.6b.safetensors" - # https://huggingface.co/nunchaku-tech/nunchaku-sana/blob/main/svdq-int4_r32-sana1.6b.safetensors shared.log.debug(f'Load module: quant=Nunchaku module=transformer repo="{nunchaku_repo}" precision={nunchaku_precision} attention={shared.opts.nunchaku_attention}') - kwargs['transformer'] = nunchaku.NunchakuSanaTransformer2DModel.from_pretrained(nunchaku_repo, torch_dtype=devices.dtype) + kwargs['transformer'] = nunchaku.NunchakuSanaTransformer2DModel.from_pretrained(nunchaku_repo, torch_dtype=devices.dtype, cache_dir=cache_dir) elif model_quant.check_quant('Model'): load_args, quant_args = model_quant.get_dit_args(kwargs_copy, module='Model') kwargs['transformer'] = diffusers.SanaTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, **load_args, **quant_args) diff --git a/pipelines/model_stablecascade.py b/pipelines/model_stablecascade.py index 60d8ca87c..40d141364 100644 --- a/pipelines/model_stablecascade.py +++ b/pipelines/model_stablecascade.py @@ -162,10 +162,10 @@ def load_cascade_combined(checkpoint_info, diffusers_load_config): # Balanced offload hooks: class StableCascadeDecoderPipelineFixed(diffusers.StableCascadeDecoderPipeline): - def guidance_scale(self): + def guidance_scale(self): # pylint: disable=invalid-overridden-method return self._guidance_scale - def do_classifier_free_guidance(self): + def do_classifier_free_guidance(self): # pylint: disable=invalid-overridden-method return self._guidance_scale > 1 @torch.no_grad() diff --git a/pipelines/model_wanai.py b/pipelines/model_wanai.py index b0a7d8d85..6a9f844d5 100644 --- a/pipelines/model_wanai.py +++ b/pipelines/model_wanai.py @@ -8,6 +8,11 @@ def load_transformer(repo_id, diffusers_load_config={}, subfolder='transformer') load_args, quant_args = model_quant.get_dit_args(diffusers_load_config, module='Model', device_map=True) fn = None + if 'VACE' in repo_id: + transformer_cls = diffusers.WanVACETransformer3DModel + else: + transformer_cls = diffusers.WanTransformer3DModel + if shared.opts.sd_unet is not None and shared.opts.sd_unet != 'Default': from modules import sd_unet if shared.opts.sd_unet not in list(sd_unet.unet_dict): @@ -20,7 +25,7 @@ def load_transformer(repo_id, diffusers_load_config={}, subfolder='transformer') transformer = None elif fn is not None and 'safetensors' in fn.lower(): shared.log.debug(f'Load model: type=WanAI {subfolder}="{fn}" quant="{model_quant.get_quant(repo_id)}" args={load_args}') - transformer = diffusers.WanTransformer3DModel.from_single_file( + transformer = transformer_cls.from_single_file( fn, cache_dir=shared.opts.hfcache_dir, **load_args, @@ -28,7 +33,7 @@ def load_transformer(repo_id, diffusers_load_config={}, subfolder='transformer') ) else: shared.log.debug(f'Load model: type=WanAI {subfolder}="{repo_id}" quant="{model_quant.get_quant_type(quant_args)}" args={load_args}') - transformer = diffusers.WanTransformer3DModel.from_pretrained( + transformer = transformer_cls.from_pretrained( repo_id, subfolder=subfolder, cache_dir=shared.opts.hfcache_dir, @@ -60,16 +65,20 @@ def load_wan(checkpoint_info, diffusers_load_config={}): repo_id = sd_models.path_to_repo(checkpoint_info) sd_models.hf_auth_check(checkpoint_info) - if 'a14b' in repo_id.lower(): + boundary_ratio = None + if 'a14b' in repo_id.lower() or 'fun-14b' in repo_id.lower(): if shared.opts.model_wan_stage == 'high noise' or shared.opts.model_wan_stage == 'first': transformer = load_transformer(repo_id, diffusers_load_config, 'transformer') transformer_2 = None + boundary_ratio = 0.0 elif shared.opts.model_wan_stage == 'low noise' or shared.opts.model_wan_stage == 'second': - transformer = load_transformer(repo_id, diffusers_load_config, 'transformer_2') - transformer_2 = None + transformer = None + transformer_2 = load_transformer(repo_id, diffusers_load_config, 'transformer_2') + boundary_ratio = 1.0 elif shared.opts.model_wan_stage == 'combined' or shared.opts.model_wan_stage == 'both': transformer = load_transformer(repo_id, diffusers_load_config, 'transformer') transformer_2 = load_transformer(repo_id, diffusers_load_config, 'transformer_2') + boundary_ratio = shared.opts.model_wan_boundary else: shared.log.error(f'Load model: type=WanAI stage="{shared.opts.model_wan_stage}" unsupported') return None @@ -80,16 +89,22 @@ def load_wan(checkpoint_info, diffusers_load_config={}): text_encoder = load_text_encoder(repo_id, diffusers_load_config) load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, module='Model') - boundary_ratio = shared.opts.model_wan_boundary if transformer_2 is not None else None if 'Wan2.2-I2V' in repo_id: - cls = diffusers.WanImageToVideoPipeline + pipe_cls = diffusers.WanImageToVideoPipeline diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["wanai"] = diffusers.WanImageToVideoPipeline + elif 'Wan2.2-VACE' in repo_id: + pipe_cls = diffusers.WanVACEPipeline + diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["wanai"] = diffusers.WanVACEPipeline + diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["wanai"] = diffusers.WanVACEPipeline + diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["wanai"] = diffusers.WanVACEPipeline else: - cls = diffusers.WanPipeline + from pipelines.wan.wan_image import WanImagePipeline + pipe_cls = diffusers.WanPipeline diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["wanai"] = diffusers.WanPipeline - shared.log.debug(f'Load model: type=WanAI model="{checkpoint_info.name}" repo="{repo_id}" cls={cls.__name__} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args} stage="{shared.opts.model_wan_stage}" boundary={boundary_ratio}') - pipe = cls.from_pretrained( + diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["wanai"] = WanImagePipeline + shared.log.debug(f'Load model: type=WanAI model="{checkpoint_info.name}" repo="{repo_id}" cls={pipe_cls.__name__} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args} stage="{shared.opts.model_wan_stage}" boundary={boundary_ratio}') + pipe = pipe_cls.from_pretrained( repo_id, transformer=transformer, transformer_2=transformer_2, diff --git a/pipelines/model_xomni.py b/pipelines/model_xomni.py new file mode 100644 index 000000000..838a0c0b3 --- /dev/null +++ b/pipelines/model_xomni.py @@ -0,0 +1,113 @@ +import torch +import transformers +import diffusers +from modules import shared, devices, sd_models, model_quant + + +class XOmniPipeline(diffusers.DiffusionPipeline): + def __init__( + self, + tokenizer=None, + model=None, + ): + super().__init__() + self.tokenizer = tokenizer + self.model = model + self.register_modules( + tokenizer=tokenizer, + model=model, + ) + + def load( + self, + repo_id, + load_config: dict = {}, + ): + from pipelines.xomni import modeling_xomni + load_args, quant_args = model_quant.get_dit_args(load_config, module='Model', device_map=True) + shared.log.debug(f'Load model: cls=XOmniPipeline module=tokenizer repo_id="{repo_id}"') + self.tokenizer = transformers.AutoTokenizer.from_pretrained( + repo_id, + use_fast=True, + ) + shared.log.debug(f'Load model: cls=XOmniPipeline module=transformer repo_id="{repo_id}" args={load_args}') + # self.model = transformers.AutoModelForCausalLM.from_pretrained( + self.model = modeling_xomni.XOmniForCausalLM.from_pretrained( + repo_id, + # trust_remote_code=True, + cache_dir=shared.opts.hfcache_dir, + **load_args, + **quant_args, + ) + flux_repo_id = "black-forest-labs/FLUX.1-dev" + shared.log.debug(f'Load model: cls=XOmniPipeline module=vision repo_id="{flux_repo_id}"') + self.model.init_vision( + flux_repo_id, + **quant_args, + ) + self.model.set_generation_mode('image') + + def __call__( + self, + prompt: str = "", + width: int = 1024, + height: int = 1024, + seed: int = -1, + temperature: float = 1.0, + downsample_size: int = 16, + min_p: float = 0.03, + top_p: float = 1.0, + cfg_scale: float = 1.0, + ): + + if isinstance(prompt, list): + prompt = prompt[0] + token_h, token_w = height // downsample_size, width // downsample_size + image_prefix = f'{token_h} {token_w}' + generation_config = transformers.generation.GenerationConfig( + max_new_tokens=token_h * token_w, + do_sample=True, + temperature=temperature, + min_p=min_p, + top_p=top_p, + guidance_scale=cfg_scale, + suppress_tokens=self.tokenizer.convert_tokens_to_ids(self.model.config.mm_special_tokens), + ) + + # Sample inputs: + tokens = self.tokenizer( + [prompt + image_prefix], + return_tensors='pt', + padding='longest', + padding_side='left', + ) + input_ids = tokens.input_ids.to(devices.device) + attention_mask = tokens.attention_mask.to(devices.device) + negative_ids = self.tokenizer.encode( + image_prefix, + add_special_tokens=False, + return_tensors='pt', + ).to(devices.device).expand(1, -1) + + torch.manual_seed(seed) + tokens = self.model.generate( + inputs=input_ids, + attention_mask=attention_mask, + generation_config=generation_config, + negative_prompt_ids=negative_ids, + ) + + tokens = torch.nn.functional.pad(tokens, (0, 1), value=self.tokenizer.convert_tokens_to_ids('')) + torch.manual_seed(seed) + _, images = self.model.mmdecode(self.tokenizer, tokens[0], skip_special_tokens=False) + images[0].save('/tmp/xomni_out.png') + return images + + +def load_xomni(checkpoint_info, diffusers_load_config={}): + repo_id = sd_models.path_to_repo(checkpoint_info) + sd_models.hf_auth_check(checkpoint_info) + + pipe = XOmniPipeline() + pipe.load(repo_id, load_config=diffusers_load_config) + return pipe diff --git a/pipelines/qwen/__init__.py b/pipelines/qwen/__init__.py new file mode 100644 index 000000000..e1455f988 --- /dev/null +++ b/pipelines/qwen/__init__.py @@ -0,0 +1,2 @@ +from pipelines.qwen.qwen_nunchaku import load_qwen_nunchaku +from pipelines.qwen.qwen_pruning import check_qwen_pruning diff --git a/pipelines/qwen/qwen_nunchaku.py b/pipelines/qwen/qwen_nunchaku.py index 084b24a1e..4c89b7b1c 100644 --- a/pipelines/qwen/qwen_nunchaku.py +++ b/pipelines/qwen/qwen_nunchaku.py @@ -11,10 +11,14 @@ def load_qwen_nunchaku(repo_id): except Exception: shared.log.error(f'Load module: quant=Nunchaku module=transformer repo="{repo_id}" low nunchaku version') return None - if repo_id.lower().endswith('qwen-image'): + if 'pruning' in repo_id.lower() or 'distill' in repo_id.lower(): + return None + elif repo_id.lower().endswith('qwen-image'): nunchaku_repo = f"nunchaku-tech/nunchaku-qwen-image/svdq-{nunchaku_precision}_r128-qwen-image.safetensors" # r32 vs r128 elif repo_id.lower().endswith('qwen-lightning'): nunchaku_repo = f"nunchaku-tech/nunchaku-qwen-image/svdq-{nunchaku_precision}_r128-qwen-image-lightningv1.1-8steps.safetensors" # 8-step variant + elif repo_id.lower().endswith('qwen-image-edit-2509'): + nunchaku_repo = f"nunchaku-tech/nunchaku-qwen-image-edit-2509/svdq-{nunchaku_precision}_r128-qwen-image-edit-2509.safetensors" # 8-step variant elif repo_id.lower().endswith('qwen-image-edit'): nunchaku_repo = f"nunchaku-tech/nunchaku-qwen-image-edit/svdq-{nunchaku_precision}_r128-qwen-image-edit.safetensors" # 8-step variant elif repo_id.lower().endswith('qwen-lightning-edit'): @@ -23,6 +27,11 @@ def load_qwen_nunchaku(repo_id): shared.log.error(f'Load module: quant=Nunchaku module=transformer repo="{repo_id}" unsupported') if nunchaku_repo is not None: shared.log.debug(f'Load module: quant=Nunchaku module=transformer repo="{nunchaku_repo}" precision={nunchaku_precision} offload={shared.opts.nunchaku_offload} attention={shared.opts.nunchaku_attention}') - transformer = NunchakuQwenImageTransformer2DModel.from_pretrained(nunchaku_repo, offload=shared.opts.nunchaku_offload, torch_dtype=devices.dtype) # pylint: disable=no-member + transformer = NunchakuQwenImageTransformer2DModel.from_pretrained( + nunchaku_repo, + offload=shared.opts.nunchaku_offload, + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + ) # pylint: disable=no-member transformer.quantization_method = 'SVDQuant' return transformer diff --git a/pipelines/qwen/qwen_pruning.py b/pipelines/qwen/qwen_pruning.py new file mode 100644 index 000000000..e2a642af1 --- /dev/null +++ b/pipelines/qwen/qwen_pruning.py @@ -0,0 +1,12 @@ +def check_qwen_pruning(repo_id, subfolder): + from modules.shared import log + if 'pruning' not in repo_id.lower(): + return repo_id, subfolder + if '2509' in (repo_id or '') or '2509' in (subfolder or ''): + repo_id, subfolder = "Qwen/Qwen-Image-Edit-2509", None + elif 'Edit' in (repo_id or '') or 'Edit' in (subfolder or ''): + repo_id, subfolder = "Qwen/Qwen-Image-Edit", None + else: + repo_id, subfolder = "Qwen/Qwen-Image", None + log.debug(f'Load model: variant=pruning target="{repo_id}"') + return repo_id, subfolder diff --git a/pipelines/wan/wan_image.py b/pipelines/wan/wan_image.py new file mode 100644 index 000000000..bd9923e5b --- /dev/null +++ b/pipelines/wan/wan_image.py @@ -0,0 +1,119 @@ +from typing import Any, Callable, Dict, List, Optional, Union +import torch +import diffusers +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import PipelineImageInput + +from modules import devices + + +class WanImagePipeline(diffusers.WanPipeline): + def __call__( + self, + prompt: Union[str, List[str]] = None, + negative_prompt: Union[str, List[str]] = None, + height: int = 480, + width: int = 832, + num_frames: int = 81, + num_inference_steps: int = 50, + guidance_scale: float = 5.0, + guidance_scale_2: Optional[float] = None, + num_videos_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + output_type: Optional[str] = "np", + return_dict: bool = True, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + strength: float = 0.3, # new + image: PipelineImageInput = None, # new + ): + # get img2img timesteps + self.scheduler.set_timesteps(num_inference_steps, device=devices.device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength) + # monkey patch original pipeline + self.scheduler.timesteps = timesteps + # self.scheduler._step_index = 0 + self.scheduler.orig_set_timesteps = self.scheduler.set_timesteps + self.scheduler.set_timesteps = lambda *args, **kwargs: None + + # prepare latents + latents = self.img2img_prepare_latents( + image=image, + timesteps=timesteps, + dtype=devices.dtype, + device=devices.device, + generator=generator, + ) + + # call original pipeline + result = super().__call__( # pylint: disable=no-member + prompt=prompt, + negative_prompt=negative_prompt, + height=height, + width=width, + num_frames=num_frames, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + guidance_scale_2=guidance_scale_2, + num_videos_per_prompt=num_videos_per_prompt, + generator=generator, + latents=latents, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + output_type=output_type, + return_dict=return_dict, + attention_kwargs=attention_kwargs, + callback_on_step_end=callback_on_step_end, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + max_sequence_length=max_sequence_length, + ) + + # un-monkey patch original pipeline + self.scheduler.set_timesteps = self.scheduler.orig_set_timesteps + return result + + def get_timesteps(self, num_inference_steps, strength): + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + # self.scheduler.set_begin_index(t_start * self.scheduler.order) + self.scheduler.set_begin_index(0) + return timesteps, num_inference_steps - t_start + + def img2img_prepare_latents( + self, + image: torch.Tensor = None, + timesteps: torch.Tensor = None, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> torch.Tensor: + from diffusers.utils.torch_utils import randn_tensor + from diffusers.video_processor import VideoProcessor + + if isinstance(image, list): + image = image[0] # ignore batch for now + + video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) + image_tensor = video_processor.preprocess(image, None, None) # convert PIL to [B, C, H, W] # channels may need rearrange + image_tensor = image_tensor.squeeze(0).to(device=device, dtype=dtype) + image_tensor = image_tensor[None, :, None, :, :] # expand before encode to [B, C, N, H, W] + encoder_output = self.vae.encode(image_tensor) + # init_latents = encoder_output.latent_dist.mode() # argmax or sample? + init_latents = encoder_output.latent_dist.sample(generator) + + latents_mean = torch.tensor(self.vae.config.latents_mean, device=device, dtype=torch.float32).view(1, self.vae.config.z_dim, 1, 1, 1) + latents_std = 1.0 / torch.tensor(self.vae.config.latents_std, device=device, dtype=torch.float32).view(1, self.vae.config.z_dim, 1, 1, 1) + init_latents = ((init_latents.float() - latents_mean) * latents_std).to(dtype) # normalized to standard distribution range + + init_noise = randn_tensor(init_latents.shape, generator=generator, device=device, dtype=dtype) + init_timestep = timesteps[:1] + noised_latents = self.scheduler.add_noise(init_latents, init_noise, init_timestep) + + return noised_latents diff --git a/pipelines/xomni/__init__.py b/pipelines/xomni/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pipelines/xomni/configuration_xomni.py b/pipelines/xomni/configuration_xomni.py new file mode 100644 index 000000000..dc22747f7 --- /dev/null +++ b/pipelines/xomni/configuration_xomni.py @@ -0,0 +1,25 @@ +from transformers import AutoConfig, Qwen2Config +from typing import Tuple + + +class XOmniConfig(Qwen2Config): + model_type = "x-omni" + + def __init__( + self, + num_mm_adap_layers: int = 4, + num_mm_head_layers: int = 4, + mm_vocab_size: int = 16448, + image_vocab_size: int = 16384, + mm_special_tokens: Tuple[str] = ('', '', ''), + **kwargs, + ): + super().__init__(**kwargs) + self.num_mm_adap_layers = num_mm_adap_layers + self.num_mm_head_layers = num_mm_head_layers + self.mm_vocab_size = mm_vocab_size + self.image_vocab_size = image_vocab_size + self.mm_special_tokens = mm_special_tokens + + +AutoConfig.register("x-omni", XOmniConfig) diff --git a/pipelines/xomni/modeling_siglip_flux.py b/pipelines/xomni/modeling_siglip_flux.py new file mode 100644 index 000000000..18b7b3463 --- /dev/null +++ b/pipelines/xomni/modeling_siglip_flux.py @@ -0,0 +1,841 @@ +import torch +import numpy as np + +from typing import Any, Callable, Dict, Tuple, List, Optional, Union +from diffusers import FluxTransformer2DModel +from diffusers.configuration_utils import register_to_config +from diffusers.utils import logging, USE_PEFT_BACKEND, scale_lora_layers, unscale_lora_layers +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.pipelines.flux.pipeline_flux import FluxPipeline, calculate_shift, retrieve_timesteps +from diffusers.image_processor import PipelineImageInput +from diffusers.pipelines.flux.pipeline_output import FluxPipelineOutput + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def drop_token(x, drop_prob: float = 0., training: bool = False, scale_by_keep: bool = True): + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0], x.shape[1], 1) + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0 and scale_by_keep: + random_tensor.div_(keep_prob) + return x * random_tensor + + +class FluxTransformer2DModelWithSigLIP(FluxTransformer2DModel): + @register_to_config + def __init__( + self, + patch_size: int = 1, + in_channels: int = 64, + out_channels: Optional[int] = None, + num_layers: int = 19, + num_single_layers: int = 38, + attention_head_dim: int = 128, + num_attention_heads: int = 24, + joint_attention_dim: int = 4096, + pooled_projection_dim: int = 768, + guidance_embeds: bool = False, + axes_dims_rope: Tuple[int] = (16, 56, 56), + siglip_channels: Optional[int] = None, + drop_token_prob: float = 0., + ): + super().__init__( + patch_size=patch_size, + in_channels=in_channels, + out_channels=out_channels, + num_layers=num_layers, + num_single_layers=num_single_layers, + attention_head_dim=attention_head_dim, + num_attention_heads=num_attention_heads, + joint_attention_dim=joint_attention_dim, + pooled_projection_dim=pooled_projection_dim, + guidance_embeds=guidance_embeds, + axes_dims_rope=axes_dims_rope, + ) + self.drop_token_prob = drop_token_prob + if siglip_channels is not None: + self.init_siglip_embed(siglip_channels) + + def init_siglip_embed(self, siglip_channels): + self.siglip_embed = torch.nn.Linear(siglip_channels, self.inner_dim, bias=False) + torch.nn.init.zeros_(self.siglip_embed.weight) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + pooled_projections: torch.Tensor = None, + timestep: torch.LongTensor = None, + img_ids: torch.Tensor = None, + txt_ids: torch.Tensor = None, + guidance: torch.Tensor = None, + siglip_tensor: Optional[torch.Tensor] = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_block_samples=None, + controlnet_single_block_samples=None, + return_dict: bool = True, + controlnet_blocks_repeat: bool = False, + ) -> Union[torch.Tensor, Transformer2DModelOutput]: + """ + The [`FluxTransformer2DModel`] forward method. + + Args: + hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`): + Input `hidden_states`. + encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`): + Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. + pooled_projections (`torch.Tensor` of shape `(batch_size, projection_dim)`): Embeddings projected + from the embeddings of input conditions. + timestep ( `torch.LongTensor`): + Used to indicate denoising step. + block_controlnet_hidden_states: (`list` of `torch.Tensor`): + A list of tensors that if specified are added to the residuals of transformer blocks. + joint_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain + tuple. + + Returns: + If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a + `tuple` where the first element is the sample tensor. + """ + if joint_attention_kwargs is not None: + joint_attention_kwargs = joint_attention_kwargs.copy() + lora_scale = joint_attention_kwargs.pop("scale", 1.0) + else: + lora_scale = 1.0 + + if USE_PEFT_BACKEND: + # weight the lora layers by setting `lora_scale` for each PEFT layer + scale_lora_layers(self, lora_scale) + else: + if joint_attention_kwargs is not None and joint_attention_kwargs.get("scale", None) is not None: + logger.warning( + "Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective." + ) + + hidden_states = self.x_embedder(hidden_states) + + timestep = timestep.to(hidden_states.dtype) * 1000 + if guidance is not None: + guidance = guidance.to(hidden_states.dtype) * 1000 + else: + guidance = None + + temb = ( + self.time_text_embed(timestep, pooled_projections) + if guidance is None + else self.time_text_embed(timestep, guidance, pooled_projections) + ) + encoder_hidden_states = self.context_embedder(encoder_hidden_states) + + if txt_ids.ndim == 3: + logger.warning( + "Passing `txt_ids` 3d torch.Tensor is deprecated." + "Please remove the batch dimension and pass it as a 2d torch Tensor" + ) + txt_ids = txt_ids[0] + if img_ids.ndim == 3: + logger.warning( + "Passing `img_ids` 3d torch.Tensor is deprecated." + "Please remove the batch dimension and pass it as a 2d torch Tensor" + ) + img_ids = img_ids[0] + + ids = torch.cat((txt_ids, img_ids), dim=0) + image_rotary_emb = self.pos_embed(ids) + + if joint_attention_kwargs is not None and "ip_adapter_image_embeds" in joint_attention_kwargs: + ip_adapter_image_embeds = joint_attention_kwargs.pop("ip_adapter_image_embeds") + ip_hidden_states = self.encoder_hid_proj(ip_adapter_image_embeds) + joint_attention_kwargs.update({"ip_hidden_states": ip_hidden_states}) + + for index_block, block in enumerate(self.transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + encoder_hidden_states, hidden_states = self._gradient_checkpointing_func( + block, + hidden_states, + encoder_hidden_states, + temb, + image_rotary_emb, + ) + + else: + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + # controlnet residual + if controlnet_block_samples is not None: + interval_control = len(self.transformer_blocks) / len(controlnet_block_samples) + interval_control = int(np.ceil(interval_control)) + # For Xlabs ControlNet. + if controlnet_blocks_repeat: + hidden_states = ( + hidden_states + controlnet_block_samples[index_block % len(controlnet_block_samples)] + ) + else: + hidden_states = hidden_states + controlnet_block_samples[index_block // interval_control] + + if siglip_tensor is not None: + siglip_tensor = drop_token(siglip_tensor, self.drop_token_prob, training=self.training) + hidden_states = hidden_states + self.siglip_embed(siglip_tensor) + + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) + + for index_block, block in enumerate(self.single_transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func( + block, + hidden_states, + temb, + image_rotary_emb, + ) + + else: + hidden_states = block( + hidden_states=hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + # controlnet residual + if controlnet_single_block_samples is not None: + interval_control = len(self.single_transformer_blocks) / len(controlnet_single_block_samples) + interval_control = int(np.ceil(interval_control)) + hidden_states[:, encoder_hidden_states.shape[1]:, ...] = ( + hidden_states[:, encoder_hidden_states.shape[1]:, ...] + + controlnet_single_block_samples[index_block // interval_control] + ) + + hidden_states = hidden_states[:, encoder_hidden_states.shape[1]:, ...] + + hidden_states = self.norm_out(hidden_states, temb) + output = self.proj_out(hidden_states) + + if USE_PEFT_BACKEND: + # remove `lora_scale` from each PEFT layer + unscale_lora_layers(self, lora_scale) + + if not return_dict: + return (output,) + + return Transformer2DModelOutput(sample=output) + + +def teacache_forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + pooled_projections: torch.Tensor = None, + timestep: torch.LongTensor = None, + img_ids: torch.Tensor = None, + txt_ids: torch.Tensor = None, + guidance: torch.Tensor = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_block_samples=None, + controlnet_single_block_samples=None, + return_dict: bool = True, + controlnet_blocks_repeat: bool = False, + siglip_tensor: Optional[torch.Tensor] = None, +) -> Union[torch.FloatTensor, Transformer2DModelOutput]: + """ + The [`FluxTransformer2DModel`] forward method. + + Args: + hidden_states (`torch.FloatTensor` of shape `(batch size, channel, height, width)`): + Input `hidden_states`. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch size, sequence_len, embed_dims)`): + Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. + pooled_projections (`torch.FloatTensor` of shape `(batch_size, projection_dim)`): Embeddings projected + from the embeddings of input conditions. + timestep ( `torch.LongTensor`): + Used to indicate denoising step. + block_controlnet_hidden_states: (`list` of `torch.Tensor`): + A list of tensors that if specified are added to the residuals of transformer blocks. + joint_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain + tuple. + + Returns: + If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a + `tuple` where the first element is the sample tensor. + """ + if joint_attention_kwargs is not None: + joint_attention_kwargs = joint_attention_kwargs.copy() + lora_scale = joint_attention_kwargs.pop("scale", 1.0) + else: + lora_scale = 1.0 + + if USE_PEFT_BACKEND: + # weight the lora layers by setting `lora_scale` for each PEFT layer + scale_lora_layers(self, lora_scale) + else: + if joint_attention_kwargs is not None and joint_attention_kwargs.get("scale", None) is not None: + logger.warning( + "Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective." + ) + + batch_size, seq_len, channels = hidden_states.shape + device, dtype = hidden_states.device, hidden_states.dtype + hidden_states = self.x_embedder(hidden_states) + + timestep = timestep.to(hidden_states.dtype) * 1000 + if guidance is not None: + guidance = guidance.to(hidden_states.dtype) * 1000 + else: + guidance = None + + temb = ( + self.time_text_embed(timestep, pooled_projections) + if guidance is None + else self.time_text_embed(timestep, guidance, pooled_projections) + ) + encoder_hidden_states = self.context_embedder(encoder_hidden_states) + + if txt_ids.ndim == 3: + logger.warning( + "Passing `txt_ids` 3d torch.Tensor is deprecated." + "Please remove the batch dimension and pass it as a 2d torch Tensor" + ) + txt_ids = txt_ids[0] + if img_ids.ndim == 3: + logger.warning( + "Passing `img_ids` 3d torch.Tensor is deprecated." + "Please remove the batch dimension and pass it as a 2d torch Tensor" + ) + img_ids = img_ids[0] + + ids = torch.cat((txt_ids, img_ids), dim=0) + image_rotary_emb = self.pos_embed(ids) + + if joint_attention_kwargs is not None and "ip_adapter_image_embeds" in joint_attention_kwargs: + ip_adapter_image_embeds = joint_attention_kwargs.pop("ip_adapter_image_embeds") + ip_hidden_states = self.encoder_hid_proj(ip_adapter_image_embeds) + joint_attention_kwargs.update({"ip_hidden_states": ip_hidden_states}) + + if self.enable_teacache: + inp = hidden_states.clone() + temb_ = temb.clone() + modulated_inp, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.transformer_blocks[0].norm1(inp, emb=temb_) + if self.cnt == 0 or self.cnt == self.num_steps - 1: + should_calc = True + self.accumulated_rel_l1_distance = 0 + else: + coefficients = [4.98651651e+02, -2.83781631e+02, 5.58554382e+01, -3.82021401e+00, 2.64230861e-01] + rescale_func = np.poly1d(coefficients) + # rescale_func = Polynomial(coefficients.reverse()) + self.accumulated_rel_l1_distance += rescale_func(((modulated_inp - self.previous_modulated_input).abs().mean() / self.previous_modulated_input.abs().mean()).cpu().item()) + if self.accumulated_rel_l1_distance < self.rel_l1_thresh: + should_calc = False + else: + should_calc = True + self.accumulated_rel_l1_distance = 0 + self.previous_modulated_input = modulated_inp + self.cnt += 1 + if self.cnt == self.num_steps: + self.cnt = 0 + + if self.enable_teacache: + if not should_calc: + hidden_states += self.previous_residual + else: + ori_hidden_states = hidden_states.clone() + for index_block, block in enumerate(self.transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + encoder_hidden_states, hidden_states = self._gradient_checkpointing_func( + block, + hidden_states, + encoder_hidden_states, + temb, + image_rotary_emb, + ) + + else: + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + # controlnet residual + if controlnet_block_samples is not None: + interval_control = len(self.transformer_blocks) / len(controlnet_block_samples) + interval_control = int(np.ceil(interval_control)) + # For Xlabs ControlNet. + if controlnet_blocks_repeat: + hidden_states = ( + hidden_states + controlnet_block_samples[index_block % len(controlnet_block_samples)] + ) + else: + hidden_states = hidden_states + controlnet_block_samples[index_block // interval_control] + + if siglip_tensor is not None: + siglip_tensor = drop_token(siglip_tensor, self.drop_token_prob, training=self.training) + hidden_states = hidden_states + self.siglip_embed(siglip_tensor) + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) + + for index_block, block in enumerate(self.single_transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func( + block, + hidden_states, + temb, + image_rotary_emb, + ) + + else: + hidden_states = block( + hidden_states=hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + # controlnet residual + if controlnet_single_block_samples is not None: + interval_control = len(self.single_transformer_blocks) / len(controlnet_single_block_samples) + interval_control = int(np.ceil(interval_control)) + hidden_states[:, encoder_hidden_states.shape[1]:, ...] = ( + hidden_states[:, encoder_hidden_states.shape[1]:, ...] + + controlnet_single_block_samples[index_block // interval_control] + ) + + hidden_states = hidden_states[:, encoder_hidden_states.shape[1]:, ...] + self.previous_residual = hidden_states - ori_hidden_states + else: + for index_block, block in enumerate(self.transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + encoder_hidden_states, hidden_states = self._gradient_checkpointing_func( + block, + hidden_states, + encoder_hidden_states, + temb, + image_rotary_emb, + ) + else: + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + # controlnet residual + if controlnet_block_samples is not None: + interval_control = len(self.transformer_blocks) / len(controlnet_block_samples) + interval_control = int(np.ceil(interval_control)) + # For Xlabs ControlNet. + if controlnet_blocks_repeat: + hidden_states = ( + hidden_states + controlnet_block_samples[index_block % len(controlnet_block_samples)] + ) + else: + hidden_states = hidden_states + controlnet_block_samples[index_block // interval_control] + if siglip_tensor is not None: + siglip_tensor = drop_token(siglip_tensor, self.drop_token_prob, training=self.training) + hidden_states = hidden_states + self.siglip_embed(siglip_tensor) + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) + + for index_block, block in enumerate(self.single_transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func( + block, + hidden_states, + temb, + image_rotary_emb, + ) + + else: + hidden_states = block( + hidden_states=hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + # controlnet residual + if controlnet_single_block_samples is not None: + interval_control = len(self.single_transformer_blocks) / len(controlnet_single_block_samples) + interval_control = int(np.ceil(interval_control)) + hidden_states[:, encoder_hidden_states.shape[1]:, ...] = ( + hidden_states[:, encoder_hidden_states.shape[1]:, ...] + + controlnet_single_block_samples[index_block // interval_control] + ) + + hidden_states = hidden_states[:, encoder_hidden_states.shape[1]:, ...] + + hidden_states = self.norm_out(hidden_states, temb) + output = self.proj_out(hidden_states) + + if USE_PEFT_BACKEND: + # remove `lora_scale` from each PEFT layer + unscale_lora_layers(self, lora_scale) + + if not return_dict: + return (output,) + + return Transformer2DModelOutput(sample=output) + + +class FluxPipelineWithSigLIP(FluxPipeline): + + @torch.no_grad() + def __call__( + self, + siglip_tensor: torch.Tensor, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Union[str, List[str]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + true_cfg_scale: float = 1.0, + true_cfg_scale_2: float = 1.0, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 28, + sigmas: Optional[List[float]] = None, + guidance_scale: float = 3.5, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + negative_ip_adapter_image: Optional[PipelineImageInput] = None, + negative_ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + will be used instead. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is + not greater than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. + true_cfg_scale (`float`, *optional*, defaults to 1.0): + When > 1.0 and a provided `negative_prompt`, enables true classifier-free guidance. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + sigmas (`List[float]`, *optional*): + Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in + their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed + will be used. + guidance_scale (`float`, *optional*, defaults to 3.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of + IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not + provided, embeddings are computed from the `ip_adapter_image` input argument. + negative_ip_adapter_image: + (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + negative_ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of + IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not + provided, embeddings are computed from the `ip_adapter_image` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.flux.FluxPipelineOutput`] instead of a plain tuple. + joint_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`. + + Examples: + + Returns: + [`~pipelines.flux.FluxPipelineOutput`] or `tuple`: [`~pipelines.flux.FluxPipelineOutput`] if `return_dict` + is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated + images. + """ + assert true_cfg_scale == true_cfg_scale_2 + + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + max_sequence_length=max_sequence_length, + ) + + self._guidance_scale = guidance_scale + self._joint_attention_kwargs = joint_attention_kwargs + self._current_timestep = None + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + lora_scale = ( + self.joint_attention_kwargs.get("scale", None) if self.joint_attention_kwargs is not None else None + ) + has_neg_prompt = negative_prompt is not None or ( + negative_prompt_embeds is not None and negative_pooled_prompt_embeds is not None + ) + do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + ( + prompt_embeds, + pooled_prompt_embeds, + text_ids, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + prompt_embeds=prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + lora_scale=lora_scale, + ) + assert do_true_cfg + ( + negative_prompt_embeds, + negative_pooled_prompt_embeds, + _, + ) = self.encode_prompt( + prompt=negative_prompt, + prompt_2=negative_prompt_2, + prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=negative_pooled_prompt_embeds, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + lora_scale=lora_scale, + ) + + # 4. Prepare latent variables + num_channels_latents = self.transformer.config.in_channels // 4 + latents, latent_image_ids = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 5. Prepare timesteps + sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + self.scheduler.config.get("base_image_seq_len", 256), + self.scheduler.config.get("max_image_seq_len", 4096), + self.scheduler.config.get("base_shift", 0.5), + self.scheduler.config.get("max_shift", 1.15), + ) + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + device, + sigmas=sigmas, + mu=mu, + ) + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self._num_timesteps = len(timesteps) + + # handle guidance + if self.transformer.config.guidance_embeds: + guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32) + guidance = guidance.expand(latents.shape[0] * 2) + else: + guidance = None + + if (ip_adapter_image is not None or ip_adapter_image_embeds is not None) and ( + negative_ip_adapter_image is None and negative_ip_adapter_image_embeds is None + ): + negative_ip_adapter_image = np.zeros((width, height, 3), dtype=np.uint8) + negative_ip_adapter_image = [negative_ip_adapter_image] * self.transformer.encoder_hid_proj.num_ip_adapters + + elif (ip_adapter_image is None and ip_adapter_image_embeds is None) and ( + negative_ip_adapter_image is not None or negative_ip_adapter_image_embeds is not None + ): + ip_adapter_image = np.zeros((width, height, 3), dtype=np.uint8) + ip_adapter_image = [ip_adapter_image] * self.transformer.encoder_hid_proj.num_ip_adapters + + if self.joint_attention_kwargs is None: + self._joint_attention_kwargs = {} + + image_embeds = None + negative_image_embeds = None + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + ) + if negative_ip_adapter_image is not None or negative_ip_adapter_image_embeds is not None: + negative_image_embeds = self.prepare_ip_adapter_image_embeds( + negative_ip_adapter_image, + negative_ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + ) + + # 6. Denoising loop + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + self._current_timestep = t + if image_embeds is not None: + self._joint_attention_kwargs["ip_adapter_image_embeds"] = image_embeds + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latents.shape[0] * 2).to(latents.dtype) + + batch_noise_pred = self.transformer( + hidden_states=torch.cat([latents, latents], dim=0), + timestep=timestep / 1000, + guidance=guidance, + pooled_projections=torch.cat([pooled_prompt_embeds, negative_pooled_prompt_embeds.expand_as(pooled_prompt_embeds)], dim=0), + encoder_hidden_states=torch.cat([prompt_embeds, negative_prompt_embeds.expand_as(prompt_embeds)], dim=0), + txt_ids=text_ids, + img_ids=latent_image_ids, + joint_attention_kwargs=self.joint_attention_kwargs, + siglip_tensor=torch.cat([siglip_tensor, torch.zeros_like(siglip_tensor)], dim=0), + return_dict=False, + )[0] + noise_pred, neg_noise_pred = batch_noise_pred.chunk(2) + noise_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + + # compute the previous noisy sample x_t -> x_t-1 + latents_dtype = latents.dtype + latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + + if latents.dtype != latents_dtype: + if torch.backends.mps.is_available(): + # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272 + latents = latents.to(latents_dtype) + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + self._current_timestep = None + + if output_type == "latent": + image = latents + else: + latents = self._unpack_latents(latents, height, width, self.vae_scale_factor) + latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor + image = self.vae.decode(latents, return_dict=False)[0] + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return FluxPipelineOutput(images=image) diff --git a/pipelines/xomni/modeling_siglip_tokenizer.py b/pipelines/xomni/modeling_siglip_tokenizer.py new file mode 100644 index 000000000..5e99d4ff8 --- /dev/null +++ b/pipelines/xomni/modeling_siglip_tokenizer.py @@ -0,0 +1,231 @@ + + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import einsum +from torchvision import transforms + +from PIL import Image +from einops import rearrange + +from .modeling_vit import create_siglip_vit + + +def create_anyres_preprocess( + short_size=384, + long_size=1152, + patch_size=16, + random_ratio=None, + min_short_size=128, + max_aspect_ratio=3., + filtering=True +): + + def resize_and_filtering(pil_image): + pil_image = pil_image.convert('RGB') + width, height = pil_image.size + ss, ls = min(width, height), max(width, height) + aspect_ratio = ls / ss + if filtering and (ss < min_short_size or aspect_ratio > max_aspect_ratio): + return None + target_width, target_height = width, height + if random_ratio is not None: + log_ratio = torch.log(torch.tensor(random_ratio)) + sqrt_ratio = torch.exp(0.5 * torch.empty(1).uniform_(log_ratio[0], log_ratio[1])).item() + target_width = int(round(target_width * sqrt_ratio)) + target_height = int(round(target_height / sqrt_ratio)) + + ss = min(target_width, target_height) + if ss < short_size: + target_width = target_width * (short_size / ss) + target_height = target_height * (short_size / ss) + + ls = max(target_width, target_height) + if ls > long_size: + target_width = target_width * (long_size / ls) + target_height = target_height * (long_size / ls) + + target_width = int(round(target_width / patch_size)) * patch_size + target_height = int(round(target_height / patch_size)) * patch_size + pil_image = pil_image.resize((target_width, target_height), resample=Image.BICUBIC) + + to_tensor = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + ]) + return to_tensor(pil_image) + + transform = transforms.Lambda(resize_and_filtering) + return transform + + +class IBQ(nn.Module): + def __init__(self, n_e, e_dim, skip_quantization_prob=0.0, quantization_temp=2.0, beta=0.25, sane_index_shape=False, l2_norm=True): + super().__init__() + self.n_e = n_e + self.e_dim = e_dim + self.quantization_temp = quantization_temp + self.skip_quantization_prob = skip_quantization_prob + self.beta = beta + self.sane_index_shape = sane_index_shape + self.l2_norm = l2_norm + + self.embedding = nn.Embedding(self.n_e, self.e_dim) + self.embedding.weight.data.uniform_(-1.0 / self.n_e, 1.0 / self.n_e) + if self.l2_norm: + self.embedding.weight.data = F.normalize(self.embedding.weight.data, p=2, dim=-1) + + def forward(self, z, temp=None, rescale_logits=False, return_logits=False, **kwargs): + assert temp is None or temp == 1.0, "Only for interface compatible with Gumbel" + assert rescale_logits == False, "Only for interface compatible with Gumbel" + assert return_logits == False, "Only for interface compatible with Gumbel" + # reshape z -> (batch, height, width, channel) and flatten + z = rearrange(z, 'b c h w -> b h w c').contiguous() + assert z.shape[-1] == self.e_dim + z_flattened = z.view(-1, self.e_dim) + # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z + + if self.l2_norm: + z = F.normalize(z, p=2, dim=-1) + z_flattened = F.normalize(z_flattened, p=2, dim=-1) + embedding = F.normalize(self.embedding.weight, p=2, dim=-1) + else: + embedding = self.embedding.weight + + d = torch.sum(z_flattened ** 2, dim=1, keepdim=True) + \ + torch.sum(embedding**2, dim=1) - 2 * \ + torch.einsum('bd,dn->bn', z_flattened, torch.einsum('n d -> d n', embedding)) + + if self.training: + logits = -d / self.quantization_temp + soft_one_hot = F.softmax(logits, dim=1) + min_encoding_indices = soft_one_hot.max(1, keepdim=True)[1] + hard_one_hot = torch.zeros_like(logits, memory_format=torch.legacy_contiguous_format).scatter_(1, min_encoding_indices, 1.0) + one_hot = hard_one_hot - soft_one_hot.detach() + soft_one_hot + + z_q = einsum('b n, n d -> b d', one_hot, self.embedding.weight).view(z.shape) + z_q_2 = einsum('b n, n d -> b d', hard_one_hot, self.embedding.weight).view(z.shape) + + # compute loss for embedding + commit_loss = torch.mean((z_q - z) ** 2) + torch.mean((z_q_2.detach() - z) ** 2) + self.beta * \ + torch.mean((z_q_2 - z.detach()) ** 2) + else: + min_encoding_indices = torch.argmin(d, dim=1) + z_q = embedding[min_encoding_indices].view(z.shape) + commit_loss = None + + if self.training and self.skip_quantization_prob > 0.0: + z_q = torch.where( + torch.rand_like(z_q[:, 0:1, 0:1, 0:1]).expand_as(z_q) <= self.skip_quantization_prob, + z, z_q, + ) + + # reshape back to match original input shape + z_q = rearrange(z_q, 'b h w c -> b c h w').contiguous() + + if self.sane_index_shape: + min_encoding_indices = min_encoding_indices.reshape(z_q.shape[0], z_q.shape[2], z_q.shape[3]) + + return (z_q, None, min_encoding_indices), commit_loss + + def get_codebook_entry(self, indices, bhwc): + # shape specifying (batch, height, width, channel) + # get quantized latent vectors + z_q = self.embedding(indices) + + if bhwc is not None: + z_q = z_q.view(bhwc) + # reshape back to match original input shape + z_q = z_q.permute(0, 3, 1, 2).contiguous() + + return z_q + + +class ResidualBlock(nn.Module): + def __init__(self, channels, num_groups=32): + super().__init__() + self.conv1 = nn.Conv2d(channels, channels, 3, padding='same') + self.norm1 = nn.GroupNorm(num_groups=num_groups, num_channels=channels) + self.activate = nn.GELU() + self.conv2 = nn.Conv2d(channels, channels, 3, padding='same') + self.norm2 = nn.GroupNorm(num_groups=num_groups, num_channels=channels) + + def forward(self, x): + res = x + x = self.norm1(x) + x = self.activate(x) + x = self.conv1(x) + x = self.norm2(x) + x = self.activate(x) + x = self.conv2(x) + return x + res + + +class VQConvProjector(nn.Module): + def __init__( + self, + z_channels=1536, + codebook_size=16384, + codebook_dim=2048, + conv_layers=2, + with_norm=True, + skip_quant_prob=0.1, + ): + super().__init__() + self.quant_conv = nn.Conv2d(z_channels, codebook_dim, 1) + self.quantize = IBQ(codebook_size, codebook_dim, skip_quant_prob, sane_index_shape=True) + self.post_quant_conv = nn.Conv2d(codebook_dim, z_channels, 1) + block = ResidualBlock + self.post_conv = nn.Sequential(*[block(z_channels) for _ in range(conv_layers)]) + + def forward(self, x, h, w): + x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w) + z = self.quant_conv(x) + (z_q, _, _), codebook_loss = self.quantize(z) + z = self.post_quant_conv(z_q) + z = self.post_conv(z) + z = rearrange(z, 'b c h w -> b (h w) c') + return z, codebook_loss + + def encode(self, x, h, w): + x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w) + z = self.quant_conv(x) + (_, _, tokens), _ = self.quantize(z) + return tokens + + def decode(self, tokens, bhwc): + z_q = self.quantize.get_codebook_entry(tokens, bhwc) + z = self.post_quant_conv(z_q) + z = self.post_conv(z) + return z + + +class SiglipTokenizer(nn.Module): + def __init__( + self, + siglip_name, + siglip_path, + projector_path, + z_channels=1536, + codebook_size=16384, + codebook_dim=2048, + with_norm=True + ): + super().__init__() + self.vit = create_siglip_vit(model_name=siglip_name, path=siglip_path) + self.vqproj = VQConvProjector( + z_channels=z_channels, + codebook_size=codebook_size, + codebook_dim=codebook_dim, + with_norm=with_norm + ) + self.vqproj.load_state_dict(torch.load(projector_path, map_location='cpu'), strict=True) + + def encode(self, x): + features, (h, w), _ = self.vit(x) + tokens = self.vqproj.encode(features, h, w) + return tokens + + def decode(self, tokens, bhwc): + return self.vqproj.decode(tokens, bhwc) diff --git a/pipelines/xomni/modeling_vit.py b/pipelines/xomni/modeling_vit.py new file mode 100644 index 000000000..150571c1d --- /dev/null +++ b/pipelines/xomni/modeling_vit.py @@ -0,0 +1,699 @@ +import math +import warnings +from dataclasses import dataclass +from functools import partial +from typing import ( + Callable, Dict, Final, List, Literal, Optional, + Sequence, Set, Tuple, Type, Union, +) + +from torch.utils.checkpoint import checkpoint +import torch +import torch.nn as nn +import torch.nn.functional as F + +from timm.layers import ( + DropPath, LayerType, Mlp, PatchDropout, + PatchEmbed, resample_abs_pos_embed, +) +from timm.models._manipulate import checkpoint_seq, named_apply + +from flash_attn import flash_attn_func, flash_attn_varlen_func + + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + # Cut & paste from PyTorch official master until it's in a few official releases - RW + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0 + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " + "The distribution of values may be incorrect.", + stacklevel=2, + ) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) # noqa: E741 + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.0)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0.0, std=1.0, a=-2.0, b=2.0): + # type: (torch.Tensor, float, float, float, float) -> torch.Tensor + r"""The original timm.models.layers.weight_init.trunc_normal_ can not handle bfloat16 yet, here we first + convert the tensor to float32, apply the trunc_normal_() in float32, and then convert it back to its orignal dtype. + Fills the input Tensor with values drawn from a truncated normal distribution. The values are effectively drawn + from the normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.trunc_normal_(w) + """ + + with torch.no_grad(): + dtype = tensor.dtype + tensor_fp32 = tensor.float() + tensor_fp32 = _no_grad_trunc_normal_(tensor_fp32, mean, std, a, b) + tensor_dtype = tensor_fp32.to(dtype=dtype) + tensor.copy_(tensor_dtype) + + +def init_weights(self): + if self.pos_embed is not None: + trunc_normal_(self.pos_embed, std=self.pos_embed.shape[1] ** -0.5) + trunc_normal_(self.latent, std=self.latent_dim**-0.5) + + +def init_weights_vit_timm(module: nn.Module, name: str = "") -> None: + """ViT weight initialization, original timm impl (for reproducibility)""" + if isinstance(module, nn.Linear): + trunc_normal_(module.weight, std=0.02) + if module.bias is not None: + nn.init.zeros_(module.bias) + elif hasattr(module, "init_weights"): + module.init_weights() + + +class Attention(nn.Module): + fused_attn: Final[bool] + + def __init__( + self, + dim: int, + num_heads: int = 8, + qkv_bias: bool = False, + qk_norm: bool = False, + attn_drop: float = 0.0, + proj_drop: float = 0.0, + norm_layer: nn.Module = nn.LayerNorm, + ) -> None: + super().__init__() + assert dim % num_heads == 0, "dim should be divisible by num_heads" + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = self.head_dim**-0.5 + # self.fused_attn = use_fused_attn() + self.fused_attn = True + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity() + self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity() + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) if proj_drop > 0.0 else nn.Identity() + + def forward(self, x: torch.Tensor, cu_slens=None) -> torch.Tensor: + B, N, C = x.shape + qkv = ( + self.qkv(x) + .reshape(B, N, 3, self.num_heads, self.head_dim) + .permute(2, 0, 3, 1, 4) + ) + q, k, v = qkv.unbind(0) + q, k = self.q_norm(q), self.k_norm(k) + + if cu_slens is not None: + q = q.permute(0, 2, 1, 3) # B, num_heads, N, C -> B, N, num_heads, C + k = k.permute(0, 2, 1, 3) + v = v.permute(0, 2, 1, 3) + max_seqlen = torch.max(cu_slens[1:] - cu_slens[:-1]).item() + x = flash_attn_varlen_func( + q.squeeze(0), + k.squeeze(0), + v.squeeze(0), + cu_seqlens_q=cu_slens, + cu_seqlens_k=cu_slens, + max_seqlen_q=max_seqlen, + max_seqlen_k=max_seqlen, + softmax_scale=self.scale, + causal=False, + ) + + x = x.reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + + else: + q = q.permute(0, 2, 1, 3) # B, num_heads, N, C -> B, N, num_heads, C + k = k.permute(0, 2, 1, 3) + v = v.permute(0, 2, 1, 3) + x = flash_attn_func(q, k, v, softmax_scale=self.scale) # -> b, n, h, c + + x = x.reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class LayerScale(nn.Module): + def __init__( + self, + dim: int, + init_values: float = 1e-5, + inplace: bool = False, + ) -> None: + super().__init__() + self.inplace = inplace + self.gamma = nn.Parameter(init_values * torch.ones(dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x.mul_(self.gamma) if self.inplace else x * self.gamma + + +class Block(nn.Module): + def __init__( + self, + dim: int, + num_heads: int, + mlp_ratio: float = 4.0, + qkv_bias: bool = False, + qk_norm: bool = False, + proj_drop: float = 0.0, + attn_drop: float = 0.0, + init_values: Optional[float] = None, + drop_path: float = 0.0, + act_layer: nn.Module = nn.GELU, + norm_layer: nn.Module = nn.LayerNorm, + mlp_layer: nn.Module = Mlp, + ) -> None: + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_norm=qk_norm, + attn_drop=attn_drop, + proj_drop=proj_drop, + norm_layer=norm_layer, + ) + self.ls1 = ( + LayerScale(dim, init_values=init_values) if init_values else nn.Identity() + ) + self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + self.norm2 = norm_layer(dim) + self.mlp = mlp_layer( + in_features=dim, + hidden_features=int(dim * mlp_ratio), + act_layer=act_layer, + drop=proj_drop, + ) + self.ls2 = ( + LayerScale(dim, init_values=init_values) if init_values else nn.Identity() + ) + self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + def forward(self, x: torch.Tensor, cu_slens=None) -> torch.Tensor: + x = x + self.drop_path1(self.ls1(self.attn(self.norm1(x), cu_slens=cu_slens))) + x = x + self.drop_path2(self.ls2(self.mlp(self.norm2(x)))) + return x + + +class VisionTransformer(nn.Module): + """Vision Transformer + + A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` + - https://arxiv.org/abs/2010.11929 + """ + + dynamic_img_size: Final[bool] + + def __init__( + self, + img_size: Union[int, Tuple[int, int]] = 224, + patch_size: Union[int, Tuple[int, int]] = 16, + in_chans: int = 3, + num_classes: int = 1000, + global_pool: Literal["", "avg", "token", "map"] = "token", + embed_dim: int = 768, + depth: int = 12, + num_heads: int = 12, + mlp_ratio: float = 4.0, + qkv_bias: bool = True, + qk_norm: bool = False, + init_values: Optional[float] = None, + class_token: bool = True, + no_embed_class: bool = False, + reg_tokens: int = 0, + pre_norm: bool = False, + fc_norm: Optional[bool] = None, + dynamic_img_size: bool = False, + dynamic_img_pad: bool = False, + drop_rate: float = 0.0, + pos_drop_rate: float = 0.0, + patch_drop_rate: float = 0.0, + proj_drop_rate: float = 0.0, + attn_drop_rate: float = 0.0, + drop_path_rate: float = 0.0, + weight_init: Literal["skip", "jax", "jax_nlhb", "moco", ""] = "", + embed_layer: Callable = PatchEmbed, + norm_layer: Optional[LayerType] = None, + act_layer: Optional[LayerType] = None, + strict_img_size: bool = False, + block_fn: Type[nn.Module] = Block, + mlp_layer: Type[nn.Module] = Mlp, + ignore_head: bool = False, + ) -> None: + """ + Args: + img_size: Input image size. + patch_size: Patch size. + in_chans: Number of image input channels. + num_classes: Mumber of classes for classification head. + global_pool: Type of global pooling for final sequence (default: 'token'). + embed_dim: Transformer embedding dimension. + depth: Depth of transformer. + num_heads: Number of attention heads. + mlp_ratio: Ratio of mlp hidden dim to embedding dim. + qkv_bias: Enable bias for qkv projections if True. + init_values: Layer-scale init values (layer-scale enabled if not None). + class_token: Use class token. + no_embed_class: Don't include position embeddings for class (or reg) tokens. + reg_tokens: Number of register tokens. + fc_norm: Pre head norm after pool (instead of before), if None, enabled when global_pool == 'avg'. + drop_rate: Head dropout rate. + pos_drop_rate: Position embedding dropout rate. + attn_drop_rate: Attention dropout rate. + drop_path_rate: Stochastic depth rate. + weight_init: Weight initialization scheme. + embed_layer: Patch embedding layer. + norm_layer: Normalization layer. + act_layer: MLP activation layer. + block_fn: Transformer block layer. + """ + super().__init__() + assert global_pool in ("", "avg", "token", "map") + assert class_token or global_pool != "token" + use_fc_norm = global_pool == "avg" if fc_norm is None else fc_norm + # norm_layer = get_norm_layer(norm_layer) or partial(nn.LayerNorm, eps=1e-6) + # act_layer = get_act_layer(act_layer) or nn.GELU + norm_layer = partial(nn.LayerNorm, eps=1e-6) + act_layer = nn.GELU + + self.num_classes = num_classes + self.global_pool = global_pool + self.num_features = self.embed_dim = ( + embed_dim # num_features for consistency with other models + ) + self.num_prefix_tokens = 1 if class_token else 0 + self.num_prefix_tokens += reg_tokens + self.num_reg_tokens = reg_tokens + self.has_class_token = class_token + self.no_embed_class = ( + no_embed_class # don't embed prefix positions (includes reg) + ) + self.dynamic_img_size = dynamic_img_size + self.grad_checkpointing = False + self.ignore_head = ignore_head + + embed_args = {} + if dynamic_img_size: + # flatten deferred until after pos embed + embed_args.update(dict(strict_img_size=False, output_fmt="NHWC")) + self.patch_embed = embed_layer( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + bias=not pre_norm, # disable bias if pre-norm is used (e.g. CLIP) + dynamic_img_pad=dynamic_img_pad, + strict_img_size=strict_img_size, + **embed_args, + ) + num_patches = self.patch_embed.num_patches + + self.cls_token = ( + nn.Parameter(torch.zeros(1, 1, embed_dim)) if class_token else None + ) + self.reg_token = ( + nn.Parameter(torch.zeros(1, reg_tokens, embed_dim)) if reg_tokens else None + ) + embed_len = ( + num_patches if no_embed_class else num_patches + self.num_prefix_tokens + ) + self.pos_embed = nn.Parameter(torch.randn(1, embed_len, embed_dim) * 0.02) + self.pos_drop = nn.Dropout(p=pos_drop_rate) + if patch_drop_rate > 0: + self.patch_drop = PatchDropout( + patch_drop_rate, + num_prefix_tokens=self.num_prefix_tokens, + ) + else: + self.patch_drop = nn.Identity() + self.norm_pre = norm_layer(embed_dim) if pre_norm else nn.Identity() + + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + self.blocks = nn.Sequential( + *[ + block_fn( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_norm=qk_norm, + init_values=init_values, + proj_drop=proj_drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[i], + norm_layer=norm_layer, + act_layer=act_layer, + mlp_layer=mlp_layer, + ) + for i in range(depth) + ] + ) + + def init_weights(self, mode: Literal["jax", "jax_nlhb", "moco", ""] = "") -> None: + assert mode in ("jax", "jax_nlhb", "moco", "") + # head_bias = -math.log(self.num_classes) if "nlhb" in mode else 0.0 + trunc_normal_(self.pos_embed, std=0.02) + if self.cls_token is not None: + nn.init.normal_(self.cls_token, std=1e-6) + named_apply(init_weights_vit_timm, self) + + @torch.jit.ignore + def no_weight_decay(self) -> Set: + return {"pos_embed", "cls_token", "dist_token"} + + @torch.jit.ignore + def group_matcher(self, coarse: bool = False) -> Dict: + return dict( + stem=r"^cls_token|pos_embed|patch_embed", # stem and embed + blocks=[(r"^blocks\.(\d+)", None), (r"^norm", (99999,))], + ) + + @torch.jit.ignore + def set_grad_checkpointing(self, enable: bool = True) -> None: + self.grad_checkpointing = enable + + @torch.jit.ignore + def get_classifier(self) -> nn.Module: + return self.head + + def reset_classifier(self, num_classes: int, global_pool=None) -> None: + self.num_classes = num_classes + if global_pool is not None: + assert global_pool in ("", "avg", "token", "map") + if global_pool == "map" and self.attn_pool is None: + assert ( + False + ), "Cannot currently add attention pooling in reset_classifier()." + elif global_pool != "map " and self.attn_pool is not None: + self.attn_pool = None # remove attention pooling + self.global_pool = global_pool + self.head = ( + nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + ) + + def rescale_positional_embedding(self, out_size): + h, w = out_size + pos_embed_shape = int((self.pos_embed.shape[1]) ** 0.5) + if (h, w) == (pos_embed_shape, pos_embed_shape): + return self.pos_embed + rescaled_positional_embedding = \ + self.pos_embed.new_zeros(1, h*w, self.pos_embed.shape[2]) + pe_2d = self.pos_embed[0].T.contiguous().view(1, -1, pos_embed_shape, pos_embed_shape) + pe_2d = F.interpolate(pe_2d, out_size, mode='bilinear', align_corners=False).view(-1, h*w) + rescaled_positional_embedding[0] = pe_2d.T.contiguous() + return rescaled_positional_embedding + + def _pos_embed(self, x: torch.Tensor) -> torch.Tensor: + if self.dynamic_img_size: + B, H, W, C = x.shape + pos_embed = resample_abs_pos_embed( + self.pos_embed, + (H, W), + num_prefix_tokens=0 if self.no_embed_class else self.num_prefix_tokens, + ) + x = x.view(B, -1, C) + else: + pos_embed = self.pos_embed + + to_cat = [] + if self.cls_token is not None: + to_cat.append(self.cls_token.expand(x.shape[0], -1, -1)) + if self.reg_token is not None: + to_cat.append(self.reg_token.expand(x.shape[0], -1, -1)) + + if self.no_embed_class: + # deit-3, updated JAX (big vision) + # position embedding does not overlap with class token, add then concat + x = x + pos_embed + if to_cat: + x = torch.cat(to_cat + [x], dim=1) + else: + # original timm, JAX, and deit vit impl + # pos_embed has entry for class token, concat then add + if to_cat: + x = torch.cat(to_cat + [x], dim=1) + x = x + pos_embed + + return self.pos_drop(x) + + def _intermediate_layers( + self, + x: torch.Tensor, + n: Union[int, Sequence] = 1, + ) -> List[torch.Tensor]: + outputs, num_blocks = [], len(self.blocks) + take_indices = set( + range(num_blocks - n, num_blocks) if isinstance(n, int) else n + ) + + # forward pass + x = self.patch_embed(x) + x = self._pos_embed(x) + x = self.patch_drop(x) + x = self.norm_pre(x) + for i, blk in enumerate(self.blocks): + x = blk(x) + if i in take_indices: + outputs.append(x) + + return outputs + + def get_intermediate_layers( + self, + x: torch.Tensor, + n: Union[int, Sequence] = 1, + reshape: bool = False, + return_prefix_tokens: bool = False, + norm: bool = False, + ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]: + """Intermediate layer accessor (NOTE: This is a WIP experiment). + Inspired by DINO / DINOv2 interface + """ + # take last n blocks if n is an int, if in is a sequence, select by matching indices + outputs = self._intermediate_layers(x, n) + if norm: + outputs = [self.norm(out) for out in outputs] + prefix_tokens = [out[:, 0 : self.num_prefix_tokens] for out in outputs] + outputs = [out[:, self.num_prefix_tokens :] for out in outputs] + + if reshape: + grid_size = self.patch_embed.grid_size + outputs = [ + out.reshape(x.shape[0], grid_size[0], grid_size[1], -1) + .permute(0, 3, 1, 2) + .contiguous() + for out in outputs + ] + + if return_prefix_tokens: + return tuple(zip(outputs, prefix_tokens)) + return tuple(outputs) + + def forward_features_list(self, x_list): + x_all = [] + image_sizes = [] + for x in x_list: + bs, _, h, w = x.shape + + # fix patch size=14 in datasets + pad_h = (self.patch_embed.patch_size[0] - h % self.patch_embed.patch_size[0]) % self.patch_embed.patch_size[0] + pad_w = (self.patch_embed.patch_size[1] - w % self.patch_embed.patch_size[1]) % self.patch_embed.patch_size[1] + x = F.pad(x, (0, pad_w, 0, pad_h)) + + bs, _, h, w = x.shape + + h = h // self.patch_embed.patch_size[0] + w = w // self.patch_embed.patch_size[1] + + x = self.patch_embed(x) + x = x + self.rescale_positional_embedding(out_size=(h, w)) + x = self.patch_drop(x) + x = self.norm_pre(x) + x_all.append(x) + image_sizes.append((h, w)) + + slen = [xi.size(1) for xi in x_all] + x = torch.cat(x_all, dim=1) + + cu_indices = [0, ] + for i in slen: + cu_indices.append(cu_indices[-1] + i) + + cu_slens = torch.tensor(cu_indices, dtype=torch.int32).to(x.device) + for idx, blk in enumerate(self.blocks): + if self.grad_checkpointing and not torch.jit.is_scripting(): + x = checkpoint(blk, x, cu_slens, use_reentrant=True) + else: + x = blk(x, cu_slens=cu_slens) + feats = x.split(slen, dim=1) #[(1, slen, c)] + return feats, image_sizes + + def forward_features(self, x: torch.Tensor) -> torch.Tensor: + bs, _, h, w = x.shape + h = h // self.patch_embed.patch_size[0] + w = w // self.patch_embed.patch_size[1] + + x = self.patch_embed(x) + # x = self._pos_embed(x) + x = x + self.rescale_positional_embedding(out_size=(h, w)) + x = self.patch_drop(x) + x = self.norm_pre(x) + if self.grad_checkpointing and not torch.jit.is_scripting(): + x = checkpoint_seq(self.blocks, x) + else: + x = self.blocks(x) + return x, (h, w) + + def forward_head(self, x: torch.Tensor, pre_logits: bool = False) -> torch.Tensor: + x = self.norm(x) + if self.attn_pool is not None: + x = self.attn_pool(x) + elif self.global_pool == "avg": + x = x[:, self.num_prefix_tokens :].mean(dim=1) + elif self.global_pool: + x = x[:, 0] # class token + x = self.fc_norm(x) + x = self.head_drop(x) + return x if pre_logits else self.head(x) + + def forward(self, x, cal_attn_pool=False): + if type(x) is list: + x, image_sizes = self.forward_features_list(x) + return x, image_sizes, None + else: + x, image_sizes = self.forward_features(x) + return x, image_sizes, None + +@dataclass +class SigLIPVisionCfg: + width: int = 1152 + layers: Union[Tuple[int, int, int, int], int] = 27 + heads: int = 16 + patch_size: int = 14 + image_size: Union[Tuple[int, int], int] = 336 + global_pool: str = "map" + mlp_ratio: float = 3.7362 + class_token: bool = False + num_classes: int = 0 + use_checkpoint: bool = False + + +SigLIP_MODEL_CONFIG = { + "siglip_so400m_patch16_384": { + "image_size": 384, + "patch_size": 16, + "width": 1152, + "layers": 27, + "heads": 16, + "mlp_ratio": 3.7362, + "global_pool": "map", + "use_checkpoint": False, + }, + "siglip2_giant_patch16_384":{ + "image_size": 384, + "patch_size": 16, + "width": 1536, + "layers": 40, + "heads": 16, + "mlp_ratio": 4, + "global_pool": "map", + "use_checkpoint": False, + }, +} + + +def resize_evaclip_pos_embed(model: VisionTransformer, interpolation: str = 'bicubic'): + # interpolate position embedding + orig_size = 24 + new_size = 128 + pos_tokens = model.pos_embed + pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, model.embed_dim).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate( + pos_tokens, size=(new_size, new_size), mode=interpolation, align_corners=False) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + model.pos_embed = nn.Parameter(pos_tokens, requires_grad=True) + return model + + +def create_siglip_vit( + model_name: str = "siglip_so400m_patch14_384", + select_layer: int = -1, + path: str = "", + gradient_checkpointing: bool = False, + **kwargs, +): + vision_cfg = SigLIPVisionCfg(**SigLIP_MODEL_CONFIG[model_name]) + + if select_layer <= 0: + layers = min(vision_cfg.layers, vision_cfg.layers + select_layer + 1) + else: + layers = min(vision_cfg.layers, select_layer) + + model = VisionTransformer( + img_size=2048, + patch_size=16, + embed_dim=vision_cfg.width, + depth=layers, + num_heads=vision_cfg.heads, + mlp_ratio=vision_cfg.mlp_ratio, + class_token=vision_cfg.class_token, + global_pool=vision_cfg.global_pool, + dynamic_img_pad=False, + strict_img_size=False, + ignore_head=kwargs.get("ignore_head", False), + weight_init=kwargs.get("weight_init", "skip"), + num_classes=0 + ) + model.config = vision_cfg + state_dict = torch.load(path, map_location="cpu") + model.load_state_dict(state_dict, strict=False) + + if gradient_checkpointing: + model.set_grad_checkpointing(True) + return model diff --git a/pipelines/xomni/modeling_xomni.py b/pipelines/xomni/modeling_xomni.py new file mode 100644 index 000000000..7e46f781c --- /dev/null +++ b/pipelines/xomni/modeling_xomni.py @@ -0,0 +1,317 @@ +import os +from types import SimpleNamespace +from typing import Tuple, List, Optional, Union + +import torch +import torch.nn as nn + +from huggingface_hub import hf_hub_download +from transformers import Qwen2ForCausalLM, AutoModel, AutoModelForCausalLM +from transformers.modeling_outputs import CausalLMOutputWithPast +from transformers.models.qwen2.modeling_qwen2 import Qwen2RMSNorm, Qwen2RotaryEmbedding, Qwen2DecoderLayer, Qwen2Model, Qwen2PreTrainedModel + +from .configuration_xomni import XOmniConfig +from .modeling_siglip_tokenizer import create_anyres_preprocess, SiglipTokenizer +from .modeling_siglip_flux import FluxTransformer2DModelWithSigLIP, FluxPipelineWithSigLIP +from .modeling_vit import create_siglip_vit + + +class XOmniDecoderLayer(Qwen2DecoderLayer): + def __init__(self, config: XOmniConfig, layer_idx: int): + super().__init__(config, layer_idx) + self.layer_idx = layer_idx + self.is_lm_layer = config.num_mm_adap_layers <= layer_idx < config.num_hidden_layers - config.num_mm_head_layers + + def forward( + self, + hidden_states: torch.Tensor, + **kwargs, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + hidden_states, multimodal_mask = torch.split(hidden_states, hidden_states.shape[-1] // 2, dim=-1) + if self.is_lm_layer: + output_hidden_states, *others = super().forward(hidden_states, **kwargs) + output_hidden_states = torch.cat([output_hidden_states, multimodal_mask], dim=-1) + return output_hidden_states, *others + + # mm_hidden_states = torch.where(multimodal_mask.bool(), hidden_states, torch.zeros_like(hidden_states)) + output_hidden_states, *others = super().forward(hidden_states, **kwargs) + output_hidden_states = torch.where(multimodal_mask.bool(), output_hidden_states, hidden_states) + output_hidden_states = torch.cat([output_hidden_states, multimodal_mask], dim=-1) + return output_hidden_states, *others + + +class XOmniModel(Qwen2Model, Qwen2PreTrainedModel): + model_type = "x-omni" + config_class = XOmniConfig + + def __init__(self, config: XOmniConfig): + Qwen2PreTrainedModel.__init__(self, config) + self.padding_idx = -1 + self.vocab_size = config.vocab_size + + self.lm_embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.mm_embed_tokens = nn.Embedding(config.mm_vocab_size, config.hidden_size, self.padding_idx) + + self.layers = nn.ModuleList( + [XOmniDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self._attn_implementation = config._attn_implementation + self.lm_norm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.mm_norm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = Qwen2RotaryEmbedding(config=config) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.lm_embed_tokens + + def set_input_embeddings(self, value): + self.lm_embed_tokens = value + + def embed_tokens(self, input_ids): + (B, L), C = input_ids.shape, self.config.hidden_size + multimodal_mask = input_ids >= self.config.vocab_size + lm_input_ids = input_ids[~multimodal_mask][None, :] + mm_input_ids = input_ids[multimodal_mask][None, :] - self.config.vocab_size + lm_embeds = self.lm_embed_tokens(lm_input_ids) + mm_embeds = self.mm_embed_tokens(mm_input_ids) + + inputs_embeds = lm_embeds.new_empty((B, L, C)) + multimodal_mask = multimodal_mask[:, :, None].expand_as(inputs_embeds) + inputs_embeds[~multimodal_mask] = lm_embeds.reshape(-1) + inputs_embeds[multimodal_mask] = mm_embeds.reshape(-1) + + inputs_embeds = torch.cat([inputs_embeds, multimodal_mask.to(inputs_embeds.dtype)], dim=-1) + return inputs_embeds + + def norm(self, hidden_states): + hidden_states, multimodal_mask = torch.split(hidden_states, hidden_states.shape[-1] // 2, dim=-1) + return torch.where(multimodal_mask.bool(), self.mm_norm(hidden_states), self.lm_norm(hidden_states)) + + +class XOmniForCausalLM(Qwen2ForCausalLM): + model_type = "x-omni" + config_class = XOmniConfig + + _keys_to_ignore_on_load_missing = r'image_tokenizer\.*' + + def __init__(self, config): + super().__init__(config) + self.model = XOmniModel(config) + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + self.mm_head = nn.Linear(config.hidden_size, config.mm_vocab_size, bias=False) + + self.generation_mode = 'text' + # Initialize weights and apply final processing + self.post_init() + + @property + def device(self): + return next(iter(self.parameters())).device + + def init_vision(self, flux_pipe_path, **kwargs): + self.som_token = self.config.mm_special_tokens[0] + self.eom_token = self.config.mm_special_tokens[1] + self.img_token = self.config.mm_special_tokens[2] + + self.vision_config = SimpleNamespace(**self.config.vision_config) + self.transform_config = SimpleNamespace(**self.vision_config.transform) + self.encoder_config = SimpleNamespace(**self.vision_config.encoder) + self.decoder_config = SimpleNamespace(**self.vision_config.decoder) + + dtype_map = {'float32': torch.float32, 'float16': torch.float16, 'bfloat16': torch.bfloat16} + self.vision_dtype = dtype_map[self.vision_config.dtype] + + self.image_transform = create_anyres_preprocess(**self.vision_config.transform) + + self.encoder_config.siglip_path = os.path.join(self.name_or_path, self.encoder_config.siglip_path) if os.path.isdir(self.name_or_path) else hf_hub_download(repo_id=self.name_or_path, filename=self.encoder_config.siglip_path) + self.encoder_config.projector_path = os.path.join(self.name_or_path, self.encoder_config.projector_path) if os.path.isdir(self.name_or_path) else hf_hub_download(repo_id=self.name_or_path, filename=self.encoder_config.projector_path) + + self.image_tokenizer = SiglipTokenizer(**vars(self.encoder_config)) + self.image_tokenizer.to(self.device, self.vision_dtype) + + transformer = FluxTransformer2DModelWithSigLIP.from_pretrained( + self.name_or_path, + siglip_channels=self.encoder_config.z_channels, + torch_dtype=self.vision_dtype, + subfolder=self.decoder_config.model_path, + **kwargs, + ) + + self.decoder_pipe = FluxPipelineWithSigLIP.from_pretrained( + flux_pipe_path, + transformer=transformer, + torch_dtype=self.vision_dtype, + ) + self.decoder_pipe.set_progress_bar_config(disable=True) + + def set_generation_mode(self, mode): + assert mode in ('text', 'image'), f'Invalid generation mode: {mode}' + self.generation_mode = mode + + def mmencode(self, tokenizer, texts=None, images=None, **kwargs): + texts = texts or [] + images = images or [] + doc = '' + while len(texts) > 0 or len(images) > 0: + if len(texts) > 0: + doc += texts.pop(0) + if len(images) > 0: + doc += self.tokenize_image(images.pop(0)) + return tokenizer.encode(doc, **kwargs) + + def mmdecode(self, tokenizer, token_ids, force_text=None, **kwargs): + force_text = force_text or [] + if isinstance(token_ids, torch.Tensor): + if len(token_ids.shape) == 2: + assert token_ids.shape[0] == 1 + token_ids = token_ids[0] + assert len(token_ids.shape) == 1 + else: + if not isinstance(token_ids[0], int): + assert len(token_ids) == 1 + token_ids = token_ids[0] + assert isinstance(token_ids[0], int) + + doc = tokenizer.decode(token_ids, **kwargs) + doc = doc.replace(tokenizer.pad_token, '') + doc = doc.replace('', '') + texts, images = [], [] + text_image_chunks = doc.split(self.eom_token) + for chunk in text_image_chunks: + text, image_str = chunk.split(self.som_token) \ + if self.som_token in chunk else (chunk, '') + texts.append(text) + if self.img_token in image_str: + image_meta, token_str = image_str.split(self.img_token) + H, W = tuple(map(int, image_meta.split(' '))) + token_ids = list(map( + lambda x: int(x.split('>')[0]), + token_str.split(' 0: + image = self.detokenize_image([force_text.pop(0)], images, token_ids, (H, W)) + else: + image = self.detokenize_image(texts, images, token_ids, (H, W)) + images.append(image) + return texts, images + + @torch.no_grad() + def tokenize_image(self, image): + assert hasattr(self, 'image_tokenizer'), 'Please call "init_vision" before that.' + + image_str = self.som_token + image = self.image_transform(image) + assert image is not None, f'Unsupported image aspect ratio (max {self.transform_config.max_aspect_ratio}) or image resolution is too low (min {self.transform_config.min_short_size})' + + image = image[None, ...].to(self.device, self.vision_dtype) + tokens = self.image_tokenizer.encode(image) + B, H, W = tokens.shape + tokens = tokens.view(B, -1).cpu().tolist()[0] + token_str = ''.join(map(lambda x: ''.format(token_id=x), tokens)) + image_str = f'{self.som_token}{H} {W}{self.img_token}{token_str}{self.eom_token}' + return image_str + + @torch.no_grad() + def detokenize_image(self, texts, images, token_ids, shape): + assert hasattr(self, 'image_tokenizer'), 'Please call "init_vision" before that.' + assert len(texts) == 1 and len(images) == 0, 'Only support one image per sample.' + H, W = shape + tokens = torch.tensor(token_ids, device=self.device, dtype=torch.long) + latents = self.image_tokenizer.decode(tokens, (1, H, W, self.encoder_config.codebook_dim)) + upscale_factor = self.decoder_config.upscale_factor + latents = latents.reshape(*latents.shape[:2], -1).transpose(1, 2).contiguous() + image = self.decoder_pipe( + latents, + [texts[0]], + negative_prompt=[''], + height=H * upscale_factor, width=W * upscale_factor, + num_inference_steps=self.decoder_config.num_inference_steps, + guidance_scale=1.0, + true_cfg_scale=self.decoder_config.cfg_scale, + true_cfg_scale_2=self.decoder_config.cfg_scale_2, + ).images[0] + + + return image + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + num_logits_to_keep: int = 0, + ) -> Union[Tuple, CausalLMOutputWithPast]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + self.model.has_sliding_layers = False + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + cache_position=cache_position, + ) + + hidden_states = outputs[0] + hidden_states = hidden_states[:, -num_logits_to_keep:, :] + logits = hidden_states.new_full( + (*hidden_states.shape[:-1], self.config.vocab_size + self.config.mm_vocab_size), + torch.finfo(hidden_states.dtype).min + ) + if self.generation_mode == 'text': + logits[:, :, :self.config.vocab_size] = self.lm_head(hidden_states) + else: + logits[:, :, self.config.vocab_size:self.config.vocab_size + self.config.image_vocab_size] = self.mm_head(hidden_states)[:, :, :self.config.image_vocab_size] + + logits = logits.float() + + loss = None + if labels is not None: + # Upcast to float if we need to compute the loss to avoid potential precision issues + logits = logits.float() + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = nn.CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +AutoModel.register(XOmniConfig, XOmniModel) +AutoModelForCausalLM.register(XOmniConfig, XOmniForCausalLM) diff --git a/requirements.txt b/requirements.txt index 25fb3c837..66803df33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,8 +35,8 @@ rich==14.1.0 safetensors==0.6.2 tensordict==0.8.3 peft==0.17.1 -httpx==0.24.1 -compel==2.1.1 +httpx==0.28.1 +compel==2.2.1 torchsde==0.2.6 antlr4-python3-runtime==4.9.3 requests==2.32.4 @@ -44,13 +44,14 @@ tqdm==4.67.1 accelerate==1.10.1 opencv-contrib-python-headless==4.11.0.86 einops==0.8.1 -huggingface_hub==0.34.4 +huggingface_hub==0.35.3 numexpr==2.11.0 numpy==2.1.2 pandas==2.3.1 numba==0.61.2 protobuf==4.25.3 pytorch_lightning==2.5.5 +PyWavelets==1.9.0 urllib3==1.26.19 Pillow==10.4.0 timm==1.0.16 @@ -69,7 +70,3 @@ seam-carving ruff pylint pre-commit - -# block -torch!=2.5.0 -torchvision!=0.20.0 diff --git a/scripts/daam/__init__.py b/scripts/daam/__init__.py new file mode 100644 index 000000000..a407cfd4d --- /dev/null +++ b/scripts/daam/__init__.py @@ -0,0 +1,5 @@ +from .experiment import * +from .heatmap import * +from .hook import * +from .utils import * +from .trace import * diff --git a/scripts/daam/evaluate.py b/scripts/daam/evaluate.py new file mode 100644 index 000000000..ec9e1567b --- /dev/null +++ b/scripts/daam/evaluate.py @@ -0,0 +1,122 @@ +from collections import defaultdict +from typing import List, Union + +from scipy.optimize import linear_sum_assignment +import PIL.Image as Image +import numpy as np +import torch +import torch.nn.functional as F + + +__all__ = ['compute_iou', 'MeanEvaluator', 'load_mask', 'compute_ioa'] + + +def compute_iou(a: torch.Tensor, b: torch.Tensor) -> float: + if a.shape[0] != b.shape[0]: + a = F.interpolate(a.unsqueeze(0).unsqueeze(0).float(), size=b.shape, mode='bicubic').squeeze() + a[a < 1] = 0 + a[a >= 1] = 1 + + intersection = (a * b).sum() + union = a.sum() + b.sum() - intersection + + return (intersection / (union + 1e-8)).item() + + +def compute_ioa(a: torch.Tensor, b: torch.Tensor) -> float: + if a.shape[0] != b.shape[0]: + a = F.interpolate(a.unsqueeze(0).unsqueeze(0).float(), size=b.shape, mode='bicubic').squeeze() + a[a < 1] = 0 + a[a >= 1] = 1 + + intersection = (a * b).sum() + area = a.sum() + + return (intersection / (area + 1e-8)).item() + + +def load_mask(path: str) -> torch.Tensor: + mask = np.array(Image.open(path)) + mask = torch.from_numpy(mask).float()[:, :, 3] # use alpha channel + mask = (mask > 0).float() + + return mask + + +class UnsupervisedEvaluator: + def __init__(self, name: str = 'UnsupervisedEvaluator'): + self.name = name + self.ious = defaultdict(list) + self.num_samples = 0 + + def log_iou(self, preds: Union[torch.Tensor, List[torch.Tensor]], truth: torch.Tensor, gt_idx: int = 0, pred_idx: int = 0): + if not isinstance(preds, list): + preds = [preds] + + iou = max(compute_iou(pred, truth) for pred in preds) + self.ious[gt_idx].append((pred_idx, iou)) + + @property + def mean_iou(self) -> float: + n = max(max(self.ious), max([y[0] for x in self.ious.values() for y in x])) + 1 + iou_matrix = np.zeros((n, n)) + count_matrix = np.zeros((n, n)) + + for gt_idx, ious in self.ious.items(): + for pred_idx, iou in ious: + iou_matrix[gt_idx, pred_idx] += iou + count_matrix[gt_idx, pred_idx] += 1 + + row_ind, col_ind = linear_sum_assignment(iou_matrix, maximize=True) + return iou_matrix[row_ind, col_ind].sum() / count_matrix[row_ind, col_ind].sum() + + def increment(self): + self.num_samples += 1 + + def __len__(self) -> int: + return self.num_samples + + def __str__(self): + return f'{self.name}<{self.mean_iou:.4f} (mIoU) {len(self)} samples>' + + +class MeanEvaluator: + def __init__(self, name: str = 'MeanEvaluator'): + self.ious: List[float] = [] + self.intensities: List[float] = [] + self.name = name + + def log_iou(self, preds: Union[torch.Tensor, List[torch.Tensor]], truth: torch.Tensor): + if not isinstance(preds, list): + preds = [preds] + + self.ious.append(max(compute_iou(pred, truth) for pred in preds)) + return self + + def log_intensity(self, pred: torch.Tensor): + self.intensities.append(pred.mean().item()) + return self + + @property + def mean_iou(self) -> float: + return np.mean(self.ious) + + @property + def mean_intensity(self) -> float: + return np.mean(self.intensities) + + @property + def ci95_miou(self) -> float: + return 1.96 * np.std(self.ious) / np.sqrt(len(self.ious)) + + def __len__(self) -> int: + return max(len(self.ious), len(self.intensities)) + + def __str__(self): + return f'{self.name}<{self.mean_iou:.4f} (±{self.ci95_miou:.3f} mIoU) {self.mean_intensity:.4f} (mInt) {len(self)} samples>' + + +if __name__ == '__main__': + mask = load_mask('truth/output/452/sink.gt.png') + + print(MeanEvaluator().log_iou(mask, mask)) diff --git a/scripts/daam/experiment.py b/scripts/daam/experiment.py new file mode 100644 index 000000000..4465a3054 --- /dev/null +++ b/scripts/daam/experiment.py @@ -0,0 +1,344 @@ +from pathlib import Path +from typing import List, Optional, Dict, Any, Union +from dataclasses import dataclass +import json + +from transformers import PreTrainedTokenizer, AutoTokenizer +import PIL.Image +import numpy as np +import torch + +from .utils import auto_autocast +from .evaluate import load_mask + + +__all__ = ['GenerationExperiment', 'COCO80_LABELS', 'COCOSTUFF27_LABELS', 'COCO80_INDICES', 'build_word_list_coco80'] + + +COCO80_LABELS: List[str] = [ + 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', + 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', + 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', + 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', + 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', + 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', + 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', + 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', + 'hair drier', 'toothbrush' +] + +COCO80_INDICES: Dict[str, int] = {x: i for i, x in enumerate(COCO80_LABELS)} + +UNUSED_LABELS: List[str] = [f'__unused_{i}__' for i in range(1, 200)] + +COCOSTUFF27_LABELS: List[str] = [ + 'electronic', 'appliance', 'food', 'furniture', 'indoor', 'kitchen', 'accessory', 'animal', 'outdoor', 'person', + 'sports', 'vehicle', 'ceiling', 'floor', 'food', 'furniture', 'rawmaterial', 'textile', 'wall', 'window', + 'building', 'ground', 'plant', 'sky', 'solid', 'structural', 'water' +] + +COCO80_ONTOLOGY = { + 'two-wheeled vehicle': ['bicycle', 'motorcycle'], + 'vehicle': ['two-wheeled vehicle', 'four-wheeled vehicle'], + 'four-wheeled vehicle': ['bus', 'truck', 'car'], + 'four-legged animals': ['livestock', 'pets', 'wild animals'], + 'livestock': ['cow', 'horse', 'sheep'], + 'pets': ['cat', 'dog'], + 'wild animals': ['elephant', 'bear', 'zebra', 'giraffe'], + 'bags': ['backpack', 'handbag', 'suitcase'], + 'sports boards': ['snowboard', 'surfboard', 'skateboard'], + 'utensils': ['fork', 'knife', 'spoon'], + 'receptacles': ['bowl', 'cup'], + 'fruits': ['banana', 'apple', 'orange'], + 'foods': ['fruits', 'meals', 'desserts'], + 'meals': ['sandwich', 'hot dog', 'pizza'], + 'desserts': ['cake', 'donut'], + 'furniture': ['chair', 'couch', 'bench'], + 'electronics': ['monitors', 'appliances'], + 'monitors': ['tv', 'cell phone', 'laptop'], + 'appliances': ['oven', 'toaster', 'refrigerator'] +} + +COCO80_TO_27 = { + 'bicycle': 'vehicle', 'car': 'vehicle', 'motorcycle': 'vehicle', 'airplane': 'vehicle', 'bus': 'vehicle', + 'train': 'vehicle', 'truck': 'vehicle', 'boat': 'vehicle', 'traffic light': 'accessory', 'fire hydrant': 'accessory', + 'stop sign': 'accessory', 'parking meter': 'accessory', 'bench': 'furniture', 'bird': 'animal', 'cat': 'animal', + 'dog': 'animal', 'horse': 'animal', 'sheep': 'animal', 'cow': 'animal', 'elephant': 'animal', 'bear': 'animal', + 'zebra': 'animal', 'giraffe': 'animal', 'backpack': 'accessory', 'umbrella': 'accessory', 'handbag': 'accessory', + 'tie': 'accessory', 'suitcase': 'accessory', 'frisbee': 'sports', 'skis': 'sports', 'snowboard': 'sports', + 'sports ball': 'sports', 'kite': 'sports', 'baseball bat': 'sports', 'baseball glove': 'sports', + 'skateboard': 'sports', 'surfboard': 'sports', 'tennis racket': 'sports', 'bottle': 'food', 'wine glass': 'food', + 'cup': 'food', 'fork': 'food', 'knife': 'food', 'spoon': 'food', 'bowl': 'food', 'banana': 'food', 'apple': 'food', + 'sandwich': 'food', 'orange': 'food', 'broccoli': 'food', 'carrot': 'food', 'hot dog': 'food', 'pizza': 'food', + 'donut': 'food', 'cake': 'food', 'chair': 'furniture', 'couch': 'furniture', 'potted plant': 'plant', + 'bed': 'furniture', 'dining table': 'furniture', 'toilet': 'furniture', 'tv': 'electronic', 'laptop': 'electronic', + 'mouse': 'electronic', 'remote': 'electronic', 'keyboard': 'electronic', 'cell phone': 'electronic', + 'microwave': 'appliance', 'oven': 'appliance', 'toaster': 'appliance', 'sink': 'appliance', + 'refrigerator': 'appliance', 'book': 'indoor', 'clock': 'indoor', 'vase': 'indoor', 'scissors': 'indoor', + 'teddy bear': 'indoor', 'hair drier': 'indoor', 'toothbrush': 'indoor' +} + + +def build_word_list_coco80() -> Dict[str, List[str]]: + words_map = COCO80_ONTOLOGY.copy() + words_map = {k: v for k, v in words_map.items() if not any(item in COCO80_ONTOLOGY for item in v)} + + return words_map + + +def _add_mask(masks: Dict[str, torch.Tensor], word: str, mask: torch.Tensor, simplify80: bool = False) -> Dict[str, torch.Tensor]: + if simplify80: + word = COCO80_TO_27.get(word, word) + + if word in masks: + masks[word] = masks[word.lower()] + mask + masks[word].clamp_(0, 1) + else: + masks[word] = mask + + return masks + + +@dataclass +class GenerationExperiment: + """Class to hold experiment parameters. Pickleable.""" + image: PIL.Image.Image + global_heat_map: torch.Tensor + prompt: str + + seed: int = None + id: str = '.' + path: Optional[Path] = None + + truth_masks: Optional[Dict[str, torch.Tensor]] = None + prediction_masks: Optional[Dict[str, torch.Tensor]] = None + annotations: Optional[Dict[str, Any]] = None + subtype: Optional[str] = '.' + tokenizer: AutoTokenizer = None + + def __post_init__(self): + if isinstance(self.path, str): + self.path = Path(self.path) + + self.path = None if self.path is None else self.path / self.id + + def nsfw(self) -> bool: + return np.sum(np.array(self.image)) == 0 + + def heat_map(self, tokenizer: AutoTokenizer = None): + if tokenizer is None: + tokenizer = self.tokenizer + + from daam import GlobalHeatMap + return GlobalHeatMap(tokenizer, self.prompt, self.global_heat_map) + + def clear_checkpoint(self): + path = self if isinstance(self, Path) else self.path + + (path / 'generation.pt').unlink(missing_ok=True) + + def save(self, path: str = None, heat_maps: bool = True, tokenizer: AutoTokenizer = None): + if path is None: + path = self.path + else: + path = Path(path) / self.id + + if tokenizer is None: + tokenizer = self.tokenizer + + (path / self.subtype).mkdir(parents=True, exist_ok=True) + torch.save(self, path / self.subtype / 'generation.pt') + self.image.save(path / self.subtype / 'output.png') + + with (path / 'prompt.txt').open('w') as f: + f.write(self.prompt) + + with (path / 'seed.txt').open('w') as f: + f.write(str(self.seed)) + + if self.truth_masks is not None: + for name, mask in self.truth_masks.items(): + im = PIL.Image.fromarray((mask * 255).unsqueeze(-1).expand(-1, -1, 4).byte().numpy()) + im.save(path / f'{name.lower()}.gt.png') + + if heat_maps and tokenizer is not None: + self.save_all_heat_maps(tokenizer) + + self.save_annotations() + + def save_annotations(self, path: Path = None): + if path is None: + path = self.path + + if self.annotations is not None: + with (path / 'annotations.json').open('w') as f: + json.dump(self.annotations, f) + + def _load_truth_masks(self, simplify80: bool = False) -> Dict[str, torch.Tensor]: + masks = {} + + for mask_path in self.path.glob('*.gt.png'): + word = mask_path.name.split('.gt.png')[0].lower() + mask = load_mask(str(mask_path)) + _add_mask(masks, word, mask, simplify80) + + return masks + + def _load_pred_masks(self, pred_prefix, composite=False, simplify80=False, vocab=None): + # type: (str, bool, bool, List[str] | None) -> Dict[str, torch.Tensor] + masks = {} + + if vocab is None: + vocab = UNUSED_LABELS + + if composite: + try: + im = PIL.Image.open(self.path / self.subtype / f'composite.{pred_prefix}.pred.png') + im = np.array(im) + + for mask_idx in np.unique(im): + mask = torch.from_numpy((im == mask_idx).astype(np.float32)) + _add_mask(masks, vocab[mask_idx], mask, simplify80) + except FileNotFoundError: + pass + else: + for mask_path in (self.path / self.subtype).glob(f'*.{pred_prefix}.pred.png'): + mask = load_mask(str(mask_path)) + word = mask_path.name.split(f'.{pred_prefix}.pred')[0].lower() + _add_mask(masks, word, mask, simplify80) + + return masks + + def clear_prediction_masks(self, name: str): + path = self if isinstance(self, Path) else self.path + path = path / self.subtype + + for mask_path in path.glob(f'*.{name}.pred.png'): + mask_path.unlink() + + def save_prediction_mask(self, mask: torch.Tensor, word: str, name: str): + path = self if isinstance(self, Path) else self.path + im = PIL.Image.fromarray((mask * 255).unsqueeze(-1).expand(-1, -1, 4).cpu().byte().numpy()) + im.save(path / self.subtype / f'{word.lower()}.{name}.pred.png') + + def save_heat_map( + self, + word: str, + tokenizer: PreTrainedTokenizer = None, + crop: int = None, + output_prefix: str = '', + absolute: bool = False + ) -> Path: + from .trace import GlobalHeatMap # because of cyclical import + + if tokenizer is None: + tokenizer = self.tokenizer + + with auto_autocast(dtype=torch.float32): + path = self.path / self.subtype / f'{output_prefix}{word.lower()}.heat_map.png' + heat_map = GlobalHeatMap(tokenizer, self.prompt, self.global_heat_map) + heat_map.compute_word_heat_map(word).expand_as(self.image, color_normalize=not absolute, out_file=path, plot=True) + + return path + + def save_all_heat_maps(self, tokenizer: PreTrainedTokenizer = None, crop: int = None) -> Dict[str, Path]: + path_map = {} + + if tokenizer is None: + tokenizer = self.tokenizer + + for word in self.prompt.split(' '): + try: + path = self.save_heat_map(word, tokenizer, crop=crop) + path_map[word] = path + except: + pass + + return path_map + + @staticmethod + def contains_truth_mask(path: Union[str, Path], prompt_id: str = None) -> bool: + if prompt_id is None: + return any(Path(path).glob('*.gt.png')) + else: + return any((Path(path) / prompt_id).glob('*.gt.png')) + + @staticmethod + def read_seed(path: Union[str, Path], prompt_id: str = None) -> int: + if prompt_id is None: + return int(Path(path).joinpath('seed.txt').read_text()) + else: + return int(Path(path).joinpath(prompt_id).joinpath('seed.txt').read_text()) + + @staticmethod + def has_annotations(path: Union[str, Path]) -> bool: + return Path(path).joinpath('annotations.json').exists() + + @staticmethod + def has_experiment(path: Union[str, Path], prompt_id: str) -> bool: + return (Path(path) / prompt_id / 'generation.pt').exists() + + @staticmethod + def read_prompt(path: Union[str, Path], prompt_id: str = None) -> str: + if prompt_id is None: + prompt_id = '.' + + with (Path(path) / prompt_id / 'prompt.txt').open('r') as f: + return f.read().strip() + + def _try_load_annotations(self): + if not (self.path / 'annotations.json').exists(): + return None + + return json.load((self.path / 'annotations.json').open()) + + def annotate(self, key: str, value: Any) -> 'GenerationExperiment': + if self.annotations is None: + self.annotations = {} + + self.annotations[key] = value + + return self + + @classmethod + def load( + cls, + path, + pred_prefix='daam', + composite=False, + simplify80=False, + vocab=None, + subtype='.', + all_subtypes=False + ): + # type: (str, str, bool, bool, List[str] | None, str, bool) -> GenerationExperiment | List[GenerationExperiment] + if all_subtypes: + experiments = [] + + for directory in Path(path).iterdir(): + if not directory.is_dir(): + continue + + try: + experiments.append(cls.load( + path, + pred_prefix=pred_prefix, + composite=composite, + simplify80=simplify80, + vocab=vocab, + subtype=directory.name + )) + except: + pass + + return experiments + + path = Path(path) + exp = torch.load(path / subtype / 'generation.pt') + exp.subtype = subtype + exp.path = path + exp.truth_masks = exp._load_truth_masks(simplify80=simplify80) + exp.prediction_masks = exp._load_pred_masks(pred_prefix, composite=composite, simplify80=simplify80, vocab=vocab) + exp.annotations = exp._try_load_annotations() + + return exp diff --git a/scripts/daam/heatmap.py b/scripts/daam/heatmap.py new file mode 100644 index 000000000..0f7a311f0 --- /dev/null +++ b/scripts/daam/heatmap.py @@ -0,0 +1,185 @@ +import io +from collections import defaultdict +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, Tuple, Set, Iterable + +from matplotlib import pyplot as plt +import numpy as np +import PIL.Image +import spacy.tokens +import torch +import torch.nn.functional as F + +from .evaluate import compute_ioa +from .utils import compute_token_merge_indices, cached_nlp, auto_autocast + +__all__ = ['GlobalHeatMap', 'RawHeatMapCollection', 'WordHeatMap', 'ParsedHeatMap', 'SyntacticHeatMapPair'] + + +def plot_overlay_heat_map(im, heat_map, word=None, out_file=None, crop=None, color_normalize=True, ax=None, cmap='jet'): + # type: (PIL.Image.Image | np.ndarray, torch.Tensor, str, Path, int, bool, plt.Axes) -> None + if ax is None: + plt.rcParams['font.size'] = 16 + plt.rcParams['figure.facecolor'] = 'black' + plt.rcParams['text.color'] = 'white' + plt.rcParams['axes.labelcolor'] = 'white' + plt.rcParams['xtick.color'] = 'black' + plt.rcParams['ytick.color'] = 'black' + plt.clf() + plt_ = plt + else: + plt_ = ax + + with auto_autocast(dtype=torch.float32): + im = np.array(im) + + if crop is not None: + heat_map = heat_map.squeeze()[crop:-crop, crop:-crop] + im = im[crop:-crop, crop:-crop] + + if color_normalize: + plt_.imshow(heat_map.squeeze().cpu().numpy(), cmap=cmap) + else: + heat_map = heat_map.clamp_(min=0, max=1) + plt_.imshow(heat_map.squeeze().cpu().numpy(), cmap=cmap, vmin=0.0, vmax=1.0) + + im = torch.from_numpy(im).float() / 255 + im = torch.cat((im, (1 - heat_map.unsqueeze(-1))), dim=-1) + plt_.imshow(im) + + if word is not None: + if ax is None: + plt.title(word) + else: + ax.set_title(word) + + if out_file is not None: + plt.savefig(out_file) + + buf = io.BytesIO() + plt.savefig(buf, format='png', bbox_inches='tight') + buf.seek(0) + image = PIL.Image.open(buf) + return image + + +class WordHeatMap: + def __init__(self, heatmap: torch.Tensor, word: str = None, word_idx: int = None): + self.word = word + self.word_idx = word_idx + self.heatmap = heatmap + + @property + def value(self): + return self.heatmap + + def plot_overlay(self, image, out_file=None, color_normalize=True, ax=None, cmap='jet', **expand_kwargs): + # type: (PIL.Image.Image | np.ndarray, Path, bool, plt.Axes, Dict[str, Any]) -> None + return plot_overlay_heat_map( + image, + self.expand_as(image, **expand_kwargs), + word=self.word, + out_file=out_file, + color_normalize=color_normalize, + ax=ax, + cmap=cmap, + ) + + def expand_as(self, image, absolute=False, threshold=None, plot=False, **plot_kwargs): + # type: (PIL.Image.Image, bool, float, bool, Dict[str, Any]) -> torch.Tensor + im = self.heatmap.unsqueeze(0).unsqueeze(0) + im = F.interpolate(im.float().detach(), size=(image.size[0], image.size[1]), mode='bicubic') + + if not absolute: + im = (im - im.min()) / (im.max() - im.min() + 1e-8) + + if threshold: + im = (im > threshold).float() + + im = im.cpu().detach().squeeze() + + if plot: + self.plot_overlay(image, **plot_kwargs) + + return im + + def compute_ioa(self, other: 'WordHeatMap'): + return compute_ioa(self.heatmap, other.heatmap) + + +@dataclass +class SyntacticHeatMapPair: + head_heat_map: WordHeatMap + dep_heat_map: WordHeatMap + head_text: str + dep_text: str + relation: str + + +@dataclass +class ParsedHeatMap: + word_heat_map: WordHeatMap + token: spacy.tokens.Token + + +class GlobalHeatMap: + def __init__(self, tokenizer: Any, prompt: str, heat_maps: torch.Tensor): + self.tokenizer = tokenizer + self.heat_maps = heat_maps + self.prompt = prompt + self.compute_word_heat_map = lru_cache(maxsize=50)(self.compute_word_heat_map) + + def compute_word_heat_map(self, word: str, word_idx: int = None, offset_idx: int = 0) -> WordHeatMap: + merge_idxs, word_idx = compute_token_merge_indices(self.tokenizer, self.prompt, word, word_idx, offset_idx) + return WordHeatMap(self.heat_maps[merge_idxs].mean(0), word, word_idx) + + def parsed_heat_maps(self) -> Iterable[ParsedHeatMap]: + for token in cached_nlp(self.prompt): + try: + heat_map = self.compute_word_heat_map(token.text) + yield ParsedHeatMap(heat_map, token) + except ValueError: + pass + + def dependency_relations(self) -> Iterable[SyntacticHeatMapPair]: + for token in cached_nlp(self.prompt): + if token.dep_ != 'ROOT': + try: + dep_heat_map = self.compute_word_heat_map(token.text) + head_heat_map = self.compute_word_heat_map(token.head.text) + + yield SyntacticHeatMapPair(head_heat_map, dep_heat_map, token.head.text, token.text, token.dep_) + except ValueError: + pass + + +RawHeatMapKey = Tuple[int, int, int] # factor, layer, head + + +class RawHeatMapCollection: + def __init__(self): + self.ids_to_heatmaps: Dict[RawHeatMapKey, torch.Tensor] = defaultdict(lambda: 0.0) + self.ids_to_num_maps: Dict[RawHeatMapKey, int] = defaultdict(lambda: 0) + + def update(self, factor: int, layer_idx: int, head_idx: int, heatmap: torch.Tensor): + with auto_autocast(dtype=torch.float32): + key = (factor, layer_idx, head_idx) + self.ids_to_heatmaps[key] = self.ids_to_heatmaps[key] + heatmap + + def factors(self) -> Set[int]: + return set(key[0] for key in self.ids_to_heatmaps.keys()) + + def layers(self) -> Set[int]: + return set(key[1] for key in self.ids_to_heatmaps.keys()) + + def heads(self) -> Set[int]: + return set(key[2] for key in self.ids_to_heatmaps.keys()) + + def __iter__(self): + return iter(self.ids_to_heatmaps.items()) + + def clear(self): + self.ids_to_heatmaps.clear() + self.ids_to_num_maps.clear() diff --git a/scripts/daam/hook.py b/scripts/daam/hook.py new file mode 100644 index 000000000..7c2866b63 --- /dev/null +++ b/scripts/daam/hook.py @@ -0,0 +1,127 @@ +from typing import List, Generic, TypeVar +import functools +import itertools + +from diffusers import UNet2DConditionModel +from diffusers.models.attention_processor import Attention +import torch.nn as nn + + +__all__ = ['ObjectHooker', 'ModuleLocator', 'AggregateHooker', 'UNetCrossAttentionLocator'] + + +ModuleType = TypeVar('ModuleType') +ModuleListType = TypeVar('ModuleListType', bound=List) + + +class ModuleLocator(Generic[ModuleType]): + def locate(self, model: nn.Module) -> List[ModuleType]: + raise NotImplementedError + + +class ObjectHooker(Generic[ModuleType]): + def __init__(self, module: ModuleType): + self.module: ModuleType = module + self.hooked = False + self.old_state = {} + + def __enter__(self): + self.hook() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.unhook() + + def hook(self): + if self.hooked: + raise RuntimeError('Already hooked module') + + self.old_state = {} + self.hooked = True + self._hook_impl() + + return self + + def unhook(self): + if not self.hooked: + raise RuntimeError('Module is not hooked') + + for k, v in self.old_state.items(): + if k.startswith('old_fn_'): + setattr(self.module, k[7:], v) + + self.hooked = False + self._unhook_impl() + + return self + + def monkey_patch(self, fn_name, fn, strict: bool = True): + try: + self.old_state[f'old_fn_{fn_name}'] = getattr(self.module, fn_name) + setattr(self.module, fn_name, functools.partial(fn, self.module)) + except AttributeError: + if strict: + raise + + def monkey_super(self, fn_name, *args, **kwargs): + return self.old_state[f'old_fn_{fn_name}'](*args, **kwargs) + + def _hook_impl(self): + raise NotImplementedError + + def _unhook_impl(self): + pass + + +class AggregateHooker(ObjectHooker[ModuleListType]): + def _hook_impl(self): + for h in self.module: + h.hook() + + def _unhook_impl(self): + for h in self.module: + h.unhook() + + def register_hook(self, hook: ObjectHooker): + self.module.append(hook) + + +class UNetCrossAttentionLocator(ModuleLocator[Attention]): + def __init__(self, restrict: bool = None, locate_middle_block: bool = False): + self.restrict = restrict + self.layer_names = [] + self.locate_middle_block = locate_middle_block + + def locate(self, model: UNet2DConditionModel) -> List[Attention]: + """ + Locate all cross-attention modules in a UNet2DConditionModel. + + Args: + model (`UNet2DConditionModel`): The model to locate the cross-attention modules in. + + Returns: + `List[Attention]`: The list of cross-attention modules. + """ + self.layer_names.clear() + blocks_list = [] + up_names = ['up'] * len(model.up_blocks) + down_names = ['down'] * len(model.down_blocks) + + for unet_block, name in itertools.chain( + zip(model.up_blocks, up_names), + zip(model.down_blocks, down_names), + zip([model.mid_block], ['mid']) if self.locate_middle_block else [], + ): + if 'CrossAttn' in unet_block.__class__.__name__: + blocks = [] + + for spatial_transformer in unet_block.attentions: + for transformer_block in spatial_transformer.transformer_blocks: + blocks.append(transformer_block.attn2) + + blocks = [b for idx, b in enumerate(blocks) if self.restrict is None or idx in self.restrict] + names = [f'{name}-attn-{i}' for i in range(len(blocks)) if self.restrict is None or i in self.restrict] + blocks_list.extend(blocks) + self.layer_names.extend(names) + + return blocks_list diff --git a/scripts/daam/trace.py b/scripts/daam/trace.py new file mode 100644 index 000000000..625e3d402 --- /dev/null +++ b/scripts/daam/trace.py @@ -0,0 +1,323 @@ +from pathlib import Path +from typing import List, Type, Any, Dict, Union +import math + +from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.attention_processor import Attention +import numpy as np +import PIL.Image as Image +import torch +import torch.nn.functional as F + +from .utils import cache_dir, auto_autocast +from .experiment import GenerationExperiment +from .heatmap import RawHeatMapCollection, GlobalHeatMap +from .hook import ObjectHooker, AggregateHooker, UNetCrossAttentionLocator + + +__all__ = ['trace', 'DiffusionHeatMapHooker', 'GlobalHeatMap'] + + +class DiffusionHeatMapHooker(AggregateHooker): + def __init__( + self, + pipeline: Union[StableDiffusionPipeline, StableDiffusionXLPipeline], + low_memory: bool = False, + load_heads: bool = False, + save_heads: bool = False, + data_dir: str = None + ): + self.all_heat_maps = RawHeatMapCollection() + h = (pipeline.unet.config.sample_size * pipeline.vae_scale_factor) + self.latent_hw = 4096 if h == 512 or h == 1024 else 9216 # 64x64 or 96x96 depending on if it's 2.0-v or 2.0 + locate_middle = load_heads or save_heads + self.locator = UNetCrossAttentionLocator(restrict={0} if low_memory else None, locate_middle_block=locate_middle) + self.last_prompt: str = '' + self.last_image: Image = None + self.time_idx = 0 + self._gen_idx = 0 + + modules = [ + UNetCrossAttentionHooker( + x, + self, + layer_idx=idx, + latent_hw=self.latent_hw, + load_heads=load_heads, + save_heads=save_heads, + data_dir=data_dir + ) for idx, x in enumerate(self.locator.locate(pipeline.unet)) + ] + + modules.append(PipelineHooker(pipeline, self)) + + if type(pipeline) == StableDiffusionXLPipeline: + modules.append(ImageProcessorHooker(pipeline.image_processor, self)) + + super().__init__(modules) + self.pipe = pipeline + + def time_callback(self, *args, **kwargs): + self.time_idx += 1 + + @property + def layer_names(self): + return self.locator.layer_names + + def to_experiment(self, path, seed=None, id='.', subtype='.', **compute_kwargs): + # type: (Union[Path, str], int, str, str, Dict[str, Any]) -> GenerationExperiment + """Exports the last generation call to a serializable generation experiment.""" + + return GenerationExperiment( + self.last_image, + self.compute_global_heat_map(**compute_kwargs).heat_maps, + self.last_prompt, + seed=seed, + id=id, + subtype=subtype, + path=path, + tokenizer=self.pipe.tokenizer, + ) + + def compute_global_heat_map(self, prompt=None, factors=None, head_idx=None, layer_idx=None, normalize=False): + # type: (str, List[float], int, int, bool) -> GlobalHeatMap + """ + Compute the global heat map for the given prompt, aggregating across time (inference steps) and space (different + spatial transformer block heat maps). + + Args: + prompt: The prompt to compute the heat map for. If none, uses the last prompt that was used for generation. + factors: Restrict the application to heat maps with spatial factors in this set. If `None`, use all sizes. + head_idx: Restrict the application to heat maps with this head index. If `None`, use all heads. + layer_idx: Restrict the application to heat maps with this layer index. If `None`, use all layers. + + Returns: + A heat map object for computing word-level heat maps. + """ + heat_maps = self.all_heat_maps + + if prompt is None: + prompt = self.last_prompt + + if factors is None: + factors = {0, 1, 2, 4, 8, 16, 32, 64} + else: + factors = set(factors) + + all_merges = [] + x = int(np.sqrt(self.latent_hw)) + + with auto_autocast(dtype=torch.float32): + for (factor, layer, head), heat_map in heat_maps: + if factor in factors and (head_idx is None or head_idx == head) and (layer_idx is None or layer_idx == layer): + heat_map = heat_map.unsqueeze(1) + # The clamping fixes undershoot. + all_merges.append(F.interpolate(heat_map, size=(x, x), mode='bicubic').clamp_(min=0)) + + try: + maps = torch.stack(all_merges, dim=0) + except RuntimeError: + if head_idx is not None or layer_idx is not None: + raise RuntimeError('No heat maps found for the given parameters.') + else: + raise RuntimeError('No heat maps found. Did you forget to call `with trace(...)` during generation?') + + maps = maps.mean(0)[:, 0] + maps = maps[:len(self.pipe.tokenizer.tokenize(prompt)) + 2] # 1 for SOS and 1 for padding + + if normalize: + maps = maps / (maps[1:-1].sum(0, keepdim=True) + 1e-6) # drop out [SOS] and [PAD] for proper probabilities + + return GlobalHeatMap(self.pipe.tokenizer, prompt, maps) + + +class ImageProcessorHooker(ObjectHooker[VaeImageProcessor]): + def __init__(self, processor: VaeImageProcessor, parent_trace: 'trace'): + super().__init__(processor) + self.parent_trace = parent_trace + + def _hooked_postprocess(hk_self, _: VaeImageProcessor, *args, **kwargs): + images = hk_self.monkey_super('postprocess', *args, **kwargs) + hk_self.parent_trace.last_image = images[0] + + return images + + def _hook_impl(self): + self.monkey_patch('postprocess', self._hooked_postprocess) + + +class PipelineHooker(ObjectHooker[StableDiffusionPipeline]): + def __init__(self, pipeline: StableDiffusionPipeline, parent_trace: 'trace'): + super().__init__(pipeline) + self.heat_maps = parent_trace.all_heat_maps + self.parent_trace = parent_trace + + def _hooked_run_safety_checker(hk_self, self: StableDiffusionPipeline, image, *args, **kwargs): + image, has_nsfw = hk_self.monkey_super('run_safety_checker', image, *args, **kwargs) + + if self.image_processor: + if torch.is_tensor(image): + images = self.image_processor.postprocess(image, output_type='pil') + else: + images = self.image_processor.numpy_to_pil(image) + else: + images = self.numpy_to_pil(image) + + hk_self.parent_trace.last_image = images[len(images)-1] + + return image, has_nsfw + + def _hooked_check_inputs(hk_self, _: StableDiffusionPipeline, prompt: Union[str, List[str]], *args, **kwargs): + if not isinstance(prompt, str) and len(prompt) > 1: + raise ValueError('Only single prompt generation is supported for heat map computation.') + elif not isinstance(prompt, str): + last_prompt = prompt[0] + else: + last_prompt = prompt + + hk_self.heat_maps.clear() + hk_self.parent_trace.last_prompt = last_prompt + + return hk_self.monkey_super('check_inputs', prompt, *args, **kwargs) + + def _hook_impl(self): + self.monkey_patch('run_safety_checker', self._hooked_run_safety_checker, strict=False) # not present in SDXL + self.monkey_patch('check_inputs', self._hooked_check_inputs) + + +class UNetCrossAttentionHooker(ObjectHooker[Attention]): + def __init__( + self, + module: Attention, + parent_trace: 'trace', + context_size: int = 77, + layer_idx: int = 0, + latent_hw: int = 9216, + load_heads: bool = False, + save_heads: bool = False, + data_dir: Union[str, Path] = None, + ): + super().__init__(module) + self.heat_maps = parent_trace.all_heat_maps + self.context_size = context_size + self.layer_idx = layer_idx + self.latent_hw = latent_hw + + self.load_heads = load_heads + self.save_heads = save_heads + self.trace = parent_trace + + if data_dir is not None: + data_dir = Path(data_dir) + else: + data_dir = cache_dir() / 'heads' + + self.data_dir = data_dir + self.data_dir.mkdir(parents=True, exist_ok=True) + + @torch.no_grad() + def _unravel_attn(self, x): + # type: (torch.Tensor) -> torch.Tensor + # x shape: (heads, height * width, tokens) + """ + Unravels the attention, returning it as a collection of heat maps. + + Args: + x (`torch.Tensor`): cross attention slice/map between the words and the tokens. + value (`torch.Tensor`): the value tensor. + + Returns: + `List[Tuple[int, torch.Tensor]]`: the list of heat maps across heads. + """ + h = w = int(math.sqrt(x.size(1))) + maps = [] + x = x.permute(2, 0, 1) + + with auto_autocast(dtype=torch.float32): + for map_ in x: + map_ = map_.view(map_.size(0), h, w) + # For Instruct Pix2Pix, divide the map into three parts: text condition, image condition and unconditional, + # and only keep the text condition part, which is first of the three parts(as per diffusers implementation). + if map_.size(0) == 24: + map_ = map_[:((map_.size(0) // 3)+1)] # Filter out unconditional and image condition + else: + map_ = map_[map_.size(0) // 2:] # # Filter out unconditional + maps.append(map_) + + maps = torch.stack(maps, 0) # shape: (tokens, heads, height, width) + return maps.permute(1, 0, 2, 3).contiguous() # shape: (heads, tokens, height, width) + + def _save_attn(self, attn_slice: torch.Tensor): + torch.save(attn_slice, self.data_dir / f'{self.trace._gen_idx}.pt') + + def _load_attn(self) -> torch.Tensor: + return torch.load(self.data_dir / f'{self.trace._gen_idx}.pt') + + def __call__( + self, + attn: Attention, + hidden_states, + encoder_hidden_states=None, + attention_mask=None, + ): + """Capture attentions and aggregate them.""" + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross is not None: + encoder_hidden_states = attn.norm_cross(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + + # DAAM save heads + if self.save_heads: + self._save_attn(attention_probs) + elif self.load_heads: + attention_probs = self._load_attn() + + # compute shape factor + factor = int(math.sqrt(self.latent_hw // attention_probs.shape[1])) + self.trace._gen_idx += 1 + + # skip if too large + if attention_probs.shape[-1] == self.context_size and factor != 8: + # shape: (batch_size, 64 // factor, 64 // factor, 77) + maps = self._unravel_attn(attention_probs) + + for head_idx, heatmap in enumerate(maps): + self.heat_maps.update(factor, self.layer_idx, head_idx, heatmap) + + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + def _hook_impl(self): + self.original_processor = self.module.processor + self.module.set_processor(self) + + def _unhook_impl(self): + self.module.set_processor(self.original_processor) + + @property + def num_heat_maps(self): + return len(next(iter(self.heat_maps.values()))) + + +trace: Type[DiffusionHeatMapHooker] = DiffusionHeatMapHooker diff --git a/scripts/daam/utils.py b/scripts/daam/utils.py new file mode 100644 index 000000000..15e2c1b5e --- /dev/null +++ b/scripts/daam/utils.py @@ -0,0 +1,107 @@ +from functools import lru_cache +from pathlib import Path +import os +import sys +import random +from typing import TypeVar + +import PIL.Image +import matplotlib.pyplot as plt +import numpy as np +import spacy +import torch + + +__all__ = ['set_seed', 'compute_token_merge_indices', 'plot_mask_heat_map', 'cached_nlp', 'cache_dir', 'auto_device', 'auto_autocast'] + + +T = TypeVar('T') + + +def auto_device(obj: T = torch.device('cpu')) -> T: + if isinstance(obj, torch.device): + return torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + if torch.cuda.is_available(): + return obj.to('cuda') + + return obj + + +def auto_autocast(*args, **kwargs): + if not torch.cuda.is_available(): + kwargs['enabled'] = False + + return torch.cuda.amp.autocast(*args, **kwargs) + + +def plot_mask_heat_map(im: PIL.Image.Image, heat_map: torch.Tensor, threshold: float = 0.4): + im = torch.from_numpy(np.array(im)).float() / 255 + mask = (heat_map.squeeze() > threshold).float() + im = im * mask.unsqueeze(-1) + plt.imshow(im) + + +def set_seed(seed: int) -> torch.Generator: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + gen = torch.Generator(device=auto_device()) + gen.manual_seed(seed) + + return gen + + +def cache_dir() -> Path: + # *nix + if os.name == 'posix' and sys.platform != 'darwin': + xdg = os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')) + return Path(xdg, 'daam') + elif sys.platform == 'darwin': + # Mac OS + return Path(os.path.expanduser('~'), 'Library/Caches/daam') + else: + # Windows + local = os.environ.get('LOCALAPPDATA', None) \ + or os.path.expanduser('~\\AppData\\Local') + return Path(local, 'daam') + + +def compute_token_merge_indices(tokenizer, prompt: str, word: str, word_idx: int = None, offset_idx: int = 0): + merge_idxs = [] + tokens = tokenizer.tokenize(prompt.lower()) + tokens = [x.replace('', '') for x in tokens] # New tokenizer uses wordpiece markers. + + if word_idx is None: + word = word.lower() + search_tokens = [x.replace('', '') for x in tokenizer.tokenize(word)] # New tokenizer uses wordpiece markers. + start_indices = [x + offset_idx for x in range(len(tokens)) if tokens[x:x + len(search_tokens)] == search_tokens] + + for indice in start_indices: + merge_idxs += [i + indice for i in range(0, len(search_tokens))] + + if not merge_idxs: + raise ValueError(f'Search word {word} not found in prompt!') + else: + merge_idxs.append(word_idx) + + return [x + 1 for x in merge_idxs], word_idx # Offset by 1. + + +nlp = None + + +@lru_cache(maxsize=100000) +def cached_nlp(prompt: str, type='en_core_web_md'): + global nlp + + if nlp is None: + try: + nlp = spacy.load(type) + except OSError: + os.system(f'python -m spacy download {type}') + nlp = spacy.load(type) + + return nlp(prompt) diff --git a/scripts/daam_ext.py b/scripts/daam_ext.py new file mode 100644 index 000000000..5ae16a855 --- /dev/null +++ b/scripts/daam_ext.py @@ -0,0 +1,57 @@ +# https://github.com/genforce/ctrl-x + +import gradio as gr +from installer import install +from modules import shared, scripts_manager, processing + + +COLORMAP = ['autumn', 'bone', 'jet', 'winter', 'rainbow', 'ocean', 'summer', 'spring', 'cool', 'hsv', 'pink', 'hot', 'parula', 'magma', 'inferno', 'plasma', 'viridis', 'cividis', 'twilight', 'shifted', 'turbo', 'deepgreen'] + + +class Script(scripts_manager.Script): + def title(self): + return 'DAAM: Diffusion Attentive Attribution Maps' + + def show(self, is_img2img): + return not is_img2img + + def ui(self, _is_img2img): + with gr.Row(): + gr.HTML('  DAAM: Diffusion Attentive Attribution Maps
') + with gr.Row(): + append_images = gr.Checkbox(label='Append heatmaps to results', value=True, elem_id='daam_append_images') + colormap = gr.Dropdown(label='Colormap', choices=COLORMAP, value='jet', type='value', elem_id='daam_colormap') + return append_images, colormap + + def run(self, p: processing.StableDiffusionProcessing, append_images, colormap): # pylint: disable=arguments-differ + c = shared.sd_model.__class__.__name__ if shared.sd_loaded else '' + if shared.sd_model_type != 'sdxl': + shared.log.warning(f'DAAM: pipeline={c} required=StableDiffusionXLPipeline') + return None + + install('thinc==8.3.4') + install('spacy==3.8.4') + + from scripts import daam # pylint: disable=no-name-in-module + orig_prompt_attention = shared.opts.prompt_attention + shared.opts.data['prompt_attention'] = 'fixed' + + # process + with daam.trace(shared.sd_model) as tc: + processed: processing.Processed = processing.process_images(p) + global_heat_map = tc.compute_global_heat_map() + shared.log.info(f'DAAM: prompt="{global_heat_map.prompt}" heatmaps={global_heat_map.heat_maps.shape}') + + # word_heat_map = global_heat_map.compute_word_heat_map('woman') + parsed_heat_maps = global_heat_map.parsed_heat_maps() + if append_images: + image = processed.images[0] + for parsed_heat_map in parsed_heat_maps: + if len(parsed_heat_map.token.text) > 1: + shared.log.debug(f'DAAM: token="{parsed_heat_map.token.text}"') + overlay = parsed_heat_map.word_heat_map.plot_overlay(image=image, color_normalize=True, cmap=colormap) + processed.images.append(overlay) + + # restore and return + shared.opts.data['prompt_attention'] = orig_prompt_attention + return processed diff --git a/scripts/infiniteyou_ext.py b/scripts/infiniteyou_ext.py index e38185aa4..677d56cca 100644 --- a/scripts/infiniteyou_ext.py +++ b/scripts/infiniteyou_ext.py @@ -13,11 +13,9 @@ orig_pipeline, orig_prompt_attention = None, None def verify_insightface(): - from installer import installed, install, install_pydantic + from installer import installed, install_insightface if not installed('insightface', reload=False, quiet=True): - install('git+https://github.com/deepinsight/insightface@554a05561cb71cfebb4e012dfea48807f845a0c2#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches - install('albumentations==1.4.3', ignore=False, reinstall=True) - install_pydantic() + install_insightface() def load_infiniteyou(model: str): diff --git a/scripts/loopback.py b/scripts/loopback.py index a22b153a2..e24683ce6 100644 --- a/scripts/loopback.py +++ b/scripts/loopback.py @@ -1,4 +1,5 @@ import math +import random import gradio as gr from modules import images, processing, scripts_manager @@ -21,10 +22,12 @@ class Script(scripts_manager.Script): final_denoising_strength = gr.Slider(minimum=0, maximum=1, step=0.01, label='Final strength', value=0.5, elem_id=self.elem_id("final_denoising_strength")) with gr.Row(): denoising_curve = gr.Dropdown(label="Strength curve", choices=["Aggressive", "Linear", "Lazy"], value="Linear") + with gr.Row(): + randomize_seed = gr.Checkbox(label="Randomize seed after each loop iteration", value=False) - return [loops, final_denoising_strength, denoising_curve] + return [loops, final_denoising_strength, denoising_curve, randomize_seed] - def run(self, p, loops, final_denoising_strength, denoising_curve): # pylint: disable=arguments-differ + def run(self, p, loops, final_denoising_strength, denoising_curve, randomize_seed): # pylint: disable=arguments-differ processing.fix_seed(p) initial_batch_count = p.n_iter p.extra_generation_params['Loopback'] = final_denoising_strength @@ -76,10 +79,14 @@ class Script(scripts_manager.Script): if initial_seed is None: initial_seed = processed.seed initial_info = processed.info + if randomize_seed: + p.seed = random.randrange(4294967294) + p.all_seeds = [p.seed] p.seed = processed.seed + 1 # why? p.denoising_strength = calculate_denoising_strength(i + 1) last_image = processed.images[0] p.init_images = [last_image] + log.info(f'Loopback: iteration={i} seed={p.seed} curve={denoising_curve} strength={p.denoising_strength}:{final_denoising_strength}') if initial_batch_count == 1: history.append(last_image) all_images.append(last_image) diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index aea0e5730..221d815dc 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -159,7 +159,7 @@ class Script(scripts_manager.Script): trust_remote_code=True, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, - _attn_implementation="eager", + # _attn_implementation="eager", **gguf_args, **quant_args, ) diff --git a/scripts/prompts_from_file.py b/scripts/prompts_from_file.py index 460b39ecf..ae2e8038f 100644 --- a/scripts/prompts_from_file.py +++ b/scripts/prompts_from_file.py @@ -147,4 +147,6 @@ class Script(scripts_manager.Script): all_negative += proc.all_negative_prompts images += proc.images infotexts += proc.infotexts + if state.interrupted: + break return get_processed(p, images, p.seed, "", all_prompts=all_prompts, all_seeds=all_seeds, all_negative_prompts=all_negative, infotexts=infotexts) diff --git a/scripts/pulid/__init__.py b/scripts/pulid/__init__.py index 958ee9c1e..864b758d1 100644 --- a/scripts/pulid/__init__.py +++ b/scripts/pulid/__init__.py @@ -8,6 +8,7 @@ from modules.errors import log sys.path.append(os.path.dirname(__file__)) try: from pulid_sdxl import StableDiffusionXLPuLIDPipeline, StableDiffusionXLPuLIDPipelineImage, StableDiffusionXLPuLIDPipelineInpaint + from pulid_flux import apply_flux, unapply_flux from pulid_utils import resize_numpy_image_long as resize import attention_processor as attention import pulid_sampling as sampling diff --git a/scripts/pulid/pulid_flux.py b/scripts/pulid/pulid_flux.py new file mode 100644 index 000000000..a419ca9c0 --- /dev/null +++ b/scripts/pulid/pulid_flux.py @@ -0,0 +1,35 @@ +from types import MethodType +import accelerate +from diffusers import FluxPipeline +from modules import shared, sd_models + + +def apply_flux(pipe: FluxPipeline): + if not hasattr(pipe, 'transformer') or not 'Nunchaku' in pipe.transformer.__class__.__name__: + shared.log.error('PuLID: flux support requires nunchaku') + return pipe + + from nunchaku.pipeline.pipeline_flux_pulid import PuLIDFluxPipeline + if not isinstance(pipe, PuLIDFluxPipeline): + from nunchaku.models.pulid.pulid_forward import pulid_forward + sd_models.clear_caches(full=True) + accelerate.hooks.remove_hook_from_module(pipe.transformer, recurse=True) + pipe = sd_models.switch_pipe(PuLIDFluxPipeline, pipe) + pipe.transformer.orig_forward = pipe.transformer.forward + pipe.transformer.forward = MethodType(pulid_forward, pipe.transformer) + pipe = sd_models.apply_balanced_offload(pipe) + pipe.pulid_model = sd_models.apply_balanced_offload(pipe.pulid_model) + shared.log.info(f'PuLID: flux applied cls={pipe.__class__.__name__} pipe={pipe.pulid_model.__class__.__name__}') + return pipe + + +def unapply_flux(pipe: FluxPipeline): + from nunchaku.pipeline.pipeline_flux_pulid import PuLIDFluxPipeline + if isinstance(pipe, PuLIDFluxPipeline) and hasattr(pipe.transformer, 'orig_forward'): + sd_models.clear_caches(full=True) + accelerate.hooks.remove_hook_from_module(pipe.transformer, recurse=True) + pipe.transformer.forward = MethodType(pipe.transformer.orig_forward, pipe.transformer) + del pipe.transformer.orig_forward + pipe = sd_models.switch_pipe(FluxPipeline, pipe) + pipe = sd_models.apply_balanced_offload(pipe) + return pipe diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index a92480824..ede691902 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -28,11 +28,9 @@ class Script(scripts_manager.Script): return True def dependencies(self): - from installer import install, installed, install_pydantic - if not installed('insightface==0.7.3', reload=False, quiet=True): - install('git+https://github.com/deepinsight/insightface@554a05561cb71cfebb4e012dfea48807f845a0c2#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches - install('albumentations==1.4.3', ignore=False, reinstall=True) - install_pydantic() + from installer import installed, install_insightface + if not installed('insightface', reload=False, quiet=True): + install_insightface() def register(self): # register xyz grid elements global registered # pylint: disable=global-statement @@ -202,14 +200,12 @@ class Script(scripts_manager.Script): errors.display(e, 'PuLID') return None elif shared.sd_model_type == 'f1': - # TODO nunchaku: pulid-f1 - shared.log.error('PuLID: f1 not supported') - return None + shared.sd_model = self.pulid.apply_flux(shared.sd_model) if shared.sd_model_type == 'sdxl': processed = self.run_sdxl(p, images, strength, zero, sampler, ortho, restore, offload, version) elif shared.sd_model_type == 'f1': - processed = None + processed = self.run_flux(p, images, strength) else: shared.log.error(f'PuLID: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={supported_model_list}') processed = None @@ -229,6 +225,12 @@ class Script(scripts_manager.Script): shared.sd_model = shared.sd_model.pipe devices.torch_gc(force=True, reason='pulid') shared.log.debug(f'PuLID complete: class={shared.sd_model.__class__.__name__} preprocess={self.preprocess:.2f} pipe={"restore" if restore else "cache"}') + if shared.sd_model_type == "f1": + restore = getattr(p, 'pulid_restore', restore) + if restore: + shared.sd_model = self.pulid.unapply_flux(shared.sd_model) + devices.torch_gc(force=True, reason='pulid') + shared.log.debug(f'PuLID complete: class={shared.sd_model.__class__.__name__} pipe={"restore" if restore else "cache"}') return processed def run_sdxl(self, p: processing.StableDiffusionProcessing, images: list, strength: float, zero: int, sampler: str, ortho: str, restore: bool, offload: bool, version: str): @@ -287,3 +289,13 @@ class Script(scripts_manager.Script): # interim = [Image.fromarray(img) for img in shared.sd_model.debug_img_list] # shared.log.debug(f'PuLID: time={t1-t0:.2f}') return processed + + def run_flux(self, p: processing.StableDiffusionProcessing, images: list, strength: float): + image = Image.fromarray(images[0]) # takes single pil image + p.task_args['id_image'] = image + p.task_args['id_weight'] = strength + shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} strength={strength} image={image}') + p.extra_generation_params["PuLID"] = f'Strength={strength}' + + processed: processing.Processed = processing.process_images(p) # runs processing using main loop + return processed diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index 76aa08b22..a0b959824 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -53,7 +53,7 @@ class Script(scripts_manager.Script): # load/download model on-demand model_path = models[model] model_name = os.path.basename(model_path) - has_checkpoint = sd_models.get_closet_checkpoint_match(model_path) + has_checkpoint = sd_models.get_closest_checkpoint_match(model_path) if has_checkpoint is None: shared.log.error(f'SVD: no checkpoint for {model_name}') modelloader.load_reference(model_path, variant='fp16') diff --git a/scripts/text2video.py b/scripts/text2video.py index b69fd7605..f6880b10f 100644 --- a/scripts/text2video.py +++ b/scripts/text2video.py @@ -62,7 +62,7 @@ class Script(scripts_manager.Script): if model['path'] in shared.opts.sd_model_checkpoint: shared.log.debug(f'Text2Video cached: model={shared.opts.sd_model_checkpoint}') else: - checkpoint = sd_models.get_closet_checkpoint_match(model['path']) + checkpoint = sd_models.get_closest_checkpoint_match(model['path']) if checkpoint is None: shared.log.debug(f'Text2Video downloading: model={model["path"]}') checkpoint = modelloader.download_diffusers_model(hub_id=model['path']) diff --git a/scripts/xyz/xyz_grid_classes.py b/scripts/xyz/xyz_grid_classes.py index 306b591f3..43dc9b2a3 100644 --- a/scripts/xyz/xyz_grid_classes.py +++ b/scripts/xyz/xyz_grid_classes.py @@ -20,6 +20,7 @@ from scripts.xyz.xyz_grid_shared import ( # pylint: disable=no-name-in-module, u apply_lora, apply_lora_strength, apply_te, + apply_guidance, apply_styles, apply_upscaler, apply_context, @@ -200,8 +201,6 @@ axis_options = [ AxisOption("[Param] Height", int, apply_field("height")), AxisOption("[Param] Seed", int, apply_seed), AxisOption("[Param] Steps", int, apply_field("steps")), - AxisOption("[Param] Guidance scale", float, apply_field("cfg_scale")), - AxisOption("[Param] Guidance end", float, apply_field("cfg_end")), AxisOption("[Param] Variation seed", int, apply_field("subseed")), AxisOption("[Param] Variation strength", float, apply_field("subseed_strength")), AxisOption("[Param] Clip skip", float, apply_clip_skip), @@ -223,12 +222,15 @@ axis_options = [ AxisOption("[Sampler] Shift", float, apply_setting("schedulers_shift")), AxisOption("[Sampler] eta delta", float, apply_setting("eta_noise_seed_delta")), AxisOption("[Sampler] eta multiplier", float, apply_setting("scheduler_eta")), + AxisOption("[Guidance] scale", float, apply_field("cfg_scale")), + AxisOption("[Guidance] end", float, apply_field("cfg_end")), + AxisOption("[Guidance] image scale", float, apply_field("image_cfg_scale")), + AxisOption("[Guidance] rescale", float, apply_field("diffusers_guidance_rescale")), + AxisOption("[Guidance] modular name", str, apply_guidance, choices=lambda: ['Default', 'CFG', 'Auto', 'Zero', 'PAG', 'APG', 'SLG', 'SEG', 'TCFG', 'FDG']), AxisOption("[Refine] Upscaler", str, apply_field("hr_upscaler"), cost=0.3, choices=lambda: [x.name for x in shared.sd_upscalers]), AxisOption("[Refine] Sampler", str, apply_hr_sampler_name, fmt=format_value_add_label, confirm=confirm_samplers, choices=lambda: [x.name for x in sd_samplers.samplers]), AxisOption("[Refine] Denoising strength", float, apply_field("denoising_strength")), AxisOption("[Refine] Hires steps", int, apply_field("hr_second_pass_steps")), - AxisOption("[Refine] Guidance scale", float, apply_field("image_cfg_scale")), - AxisOption("[Refine] Guidance rescale", float, apply_field("diffusers_guidance_rescale")), AxisOption("[Refine] Refiner start", float, apply_field("refiner_start")), AxisOption("[Refine] Refiner steps", float, apply_field("refiner_steps")), AxisOption("[Postprocess] Upscaler", str, apply_upscaler, cost=0.4, choices=lambda: [x.name for x in shared.sd_upscalers]), diff --git a/scripts/xyz/xyz_grid_shared.py b/scripts/xyz/xyz_grid_shared.py index 0c95d14de..dd454d3c2 100644 --- a/scripts/xyz/xyz_grid_shared.py +++ b/scripts/xyz/xyz_grid_shared.py @@ -164,7 +164,7 @@ def apply_sdnq_quant_te(p, x, xs): def apply_checkpoint(p, x, xs): if x == shared.opts.sd_model_checkpoint: return - info = sd_models.get_closet_checkpoint_match(x) + info = sd_models.get_closest_checkpoint_match(x) if info is None: shared.log.warning(f"XYZ grid: apply checkpoint unknown checkpoint: {x}") else: @@ -178,7 +178,7 @@ def apply_refiner(p, x, xs): return if x == 'None': return - info = sd_models.get_closet_checkpoint_match(x) + info = sd_models.get_closest_checkpoint_match(x) if info is None: shared.log.warning(f"XYZ grid: apply refiner unknown checkpoint: {x}") else: @@ -254,6 +254,13 @@ def apply_te(p, x, xs): shared.log.debug(f'XYZ grid apply text-encoder: "{x}"') +def apply_guidance(p, x, xs): + from modules.modular_guiders import guiders + guiders = list(guiders.keys()) + p.guidance_name = [g for g in guiders if g.lower().startswith(x.lower())][0] + shared.log.debug(f'XYZ grid apply guidance: "{p.guidance_name}"') + + def apply_styles(p: processing.StableDiffusionProcessingTxt2Img, x: str, _): p.styles.extend(x.split(',')) shared.log.debug(f'XYZ grid apply style: "{x}"') diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index 2ec2452da..3874deb5e 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -54,9 +54,9 @@ class Script(scripts_manager.Script): with gr.Row(): with gr.Column(): - csv_mode = gr.Checkbox(label='Text inputs', value=False, elem_id=self.elem_id("csv_mode"), container=False) - draw_legend = gr.Checkbox(label='Legend', value=True, elem_id=self.elem_id("draw_legend"), container=False) - no_fixed_seeds = gr.Checkbox(label='Random seeds', value=False, elem_id=self.elem_id("no_fixed_seeds"), container=False) + draw_legend = gr.Checkbox(label='Draw legend', value=True, elem_id=self.elem_id("draw_legend"), container=False) + csv_mode = gr.Checkbox(label='Use text inputs', value=False, elem_id=self.elem_id("csv_mode"), container=False) + no_fixed_seeds = gr.Checkbox(label='Use random seeds', value=False, elem_id=self.elem_id("no_fixed_seeds"), container=False) include_time = gr.Checkbox(label='Add time info', value=False, elem_id=self.elem_id("include_time"), container=False) include_text = gr.Checkbox(label='Add text info', value=False, elem_id=self.elem_id("include_text"), container=False) with gr.Column(): @@ -168,8 +168,6 @@ class Script(scripts_manager.Script): create_video, video_type, video_duration, video_loop, video_pad, video_interpolate, ): # pylint: disable=W0221 jobid = shared.state.begin('XYZ Grid') - if not no_fixed_seeds: - processing.fix_seed(p) if not shared.opts.return_grid: p.batch_size = 1 @@ -251,6 +249,8 @@ class Script(scripts_manager.Script): xs = fix_axis_seeds(x_opt, xs) ys = fix_axis_seeds(y_opt, ys) zs = fix_axis_seeds(z_opt, zs) + else: + processing.fix_seed(p) total_jobs = len(xs) * len(ys) * len(zs) if x_opt.label == 'Steps': diff --git a/scripts/xyz_grid_on.py b/scripts/xyz_grid_on.py index f5a910f93..595536067 100644 --- a/scripts/xyz_grid_on.py +++ b/scripts/xyz_grid_on.py @@ -324,11 +324,17 @@ class Script(scripts_manager.Script): def cell(x, y, z, ix, iy, iz): if shared.state.interrupted: + shared.log.warning('XYZ grid: Interrupted') return processing.Processed(p, [], p.seed, ""), 0 p.xyz = True pc = copy(p) pc.override_settings_restore_afterwards = False pc.styles = pc.styles[:] + if no_fixed_seeds: + pc.seed = -1 + processing.fix_seed(pc) + pc.all_seeds = None + pc.all_subseeds = None x_opt.apply(pc, x, xs) y_opt.apply(pc, y, ys) z_opt.apply(pc, z, zs) diff --git a/webui.py b/webui.py index ec33c908e..8dc29d5bc 100644 --- a/webui.py +++ b/webui.py @@ -152,9 +152,9 @@ def initialize(): def load_model(): modeldata.model_data.locked = False - if not shared.opts.sd_checkpoint_autoload and shared.cmd_opts.ckpt is None: - log.info('Model: autoload=False') - else: + autoload = shared.opts.sd_checkpoint_autoload or shared.cmd_opts.ckpt is not None + log.info(f'Model: autoload={autoload} selected="{shared.opts.sd_model_checkpoint}"') + if autoload: jobid = shared.state.begin('Load model') thread_model = Thread(target=lambda: shared.sd_model) thread_model.start() diff --git a/wiki b/wiki index 76be4e54b..78e4e6b94 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 76be4e54b2c40bac1502cf97caa4a04bec0a033f +Subproject commit 78e4e6b94adac848c118b85d7a4c552babcb1acd