diff --git a/.eslintrc.json b/.eslintrc.json index 32283e620..81c196021 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -116,7 +116,8 @@ "idbDel": "readonly", "idbAdd": "readonly", "initChangelog": "readonly", - "sendNotification": "readonly" + "sendNotification": "readonly", + "monitorConnection": "readonly" }, "ignorePatterns": [ "node_modules", diff --git a/.gitignore b/.gitignore index 9a3415c86..9bc6d5a24 100644 --- a/.gitignore +++ b/.gitignore @@ -46,14 +46,6 @@ tunableop_results*.csv !package.json !requirements.txt -# pyinstaller -*.spec -build/ -dist/ - -# dynamically generated -/repositories/deepseek-vl2/ - # all dynamic stuff /extensions/**/* /outputs/**/* diff --git a/.gitmodules b/.gitmodules index 0ead18186..41ed87300 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,10 +14,6 @@ path = extensions-builtin/stable-diffusion-webui-rembg url = https://github.com/vladmandic/sd-extension-rembg ignore = dirty -[submodule "extensions-builtin/sd-webui-agent-scheduler"] - path = extensions-builtin/sd-webui-agent-scheduler - url = https://github.com/ArtVentureX/sd-webui-agent-scheduler - ignore = dirty [submodule "extensions-builtin/sdnext-modernui"] path = extensions-builtin/sdnext-modernui url = https://github.com/BinaryQuantumSoul/sdnext-modernui diff --git a/.pylintrc b/.pylintrc index 5bd9493dd..7b6cdb7cb 100644 --- a/.pylintrc +++ b/.pylintrc @@ -44,6 +44,7 @@ ignore-paths=/usr/lib/.*$, pipelines/segmoe, scripts/consistory, scripts/ctrlx, + scripts/daam, scripts/demofusion, scripts/freescale, scripts/infiniteyou, diff --git a/.ruff.toml b/.ruff.toml index 74b734878..297515402 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -24,6 +24,7 @@ exclude = [ "pipelines/segmoe", "scripts/lbm", + "scripts/daam", "scripts/xadapter", "scripts/pulid", "scripts/instantir", diff --git a/CHANGELOG.md b/CHANGELOG.md index a14c843c2..7a68ac79b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,98 @@ # Change Log for SD.Next +## Update for 2025-10-01 + +- **Models** + - [WAN 2.2 14B VACE](https://huggingface.co/alibaba-pai/Wan2.2-VACE-Fun-A14B) + available for *text-to-image* and *text-to-video* and *image-to-video* workflows + - [Qwen Image Edit 2509](https://huggingface.co/Qwen/Qwen-Image-Edit-2509) and [Nunchaku Qwen Image Edit 2509](https://huggingface.co/nunchaku-tech/nunchaku-qwen-image-edit-2509) + updated version of Qwen Image Edit with improved image consistency + - [HiDream E1.1](https://huggingface.co/HiDream-ai/HiDream-E1-1) + updated version of E1 image editing model + - [Tencent FLUX.1 Dev SRPO](https://huggingface.co/tencent/SRPO) + SRPO is trained by Tencent with specific technique: directly aligning the full diffusion trajectory with fine-grained human preference + - [Nunchaku SDXL](https://huggingface.co/nunchaku-tech/nunchaku-sdxl) and [Nunchaku SDXL Turbo](https://huggingface.co/nunchaku-tech/nunchaku-sdxl-turbo) + impact of nunchaku engine on unet-based model such as sdxl is much less than on a dit-based models, but its still significantly faster than baseline + note that nunchaku optimized and prequantized unet is replacement for base unet, so its only applicable to base models, not any of finetunes + *how to use*: enable nunchaku in settings -> quantization and then load either sdxl-base or sdxl-base-turbo reference models + *note*: sdxl support for nunchaku is not in released version of `nunchaku==1.0.0`, so you need to build [nunchaku](https://nunchaku.tech/docs/nunchaku/installation/installation.html) from source +- **Features** + - [Cache-DiT](https://github.com/vipshop/cache-dit) + cache-dit is a unified, flexible and training-free cache acceleration framework + compatible with many dit-based models such as FLUX.1, Qwen, HunyuanImage, Wan2.2, Chroma, etc. + enable in *settings -> pipeline modifers -> cache-dit* + - [Nunchaku Flux.1 PulID](https://nunchaku.tech/docs/nunchaku/python_api/nunchaku.pipeline.pipeline_flux_pulid.html) + automatically enabled if loaded model is FLUX.1 with Nunchaku engine enabled and when PulID script is enabled +- **Compute** + - **ROCm** for Windows + support for both official torch preview release of `torch-rocm` for windows and **TheRock** unoffical `torch-rocm` builds for windows + note that rocm for windows is still in preview and has limited gpu support, please check rocm docs for details + - **DirectML** warn as end-of-life + `torch-directml` received no updates in over 1 year and its currently superceded by `rocm` or `zluda` + - command line params `--use-zluda` and `--use-rocm` will attempt desired operation or fail if not possible + previously sdnext was performing a fallback to `torch-cpu` which is not desired + - **installer**: warn if cuda or rocm are available and `torch-cpu` is installed +- **Extensions** + - [Agent-Scheduler](https://github.com/SipherAGI/sd-webui-agent-scheduler) + was a high-value built-in extension, but it has not been maintained for 1.5 years + it also does not work with control and video tabs which are the core of sdnext nowadays + so it has been removed from built-in extensions: manual installation is still possible + - [DAAM: Diffusion Attentive Attribution Maps](https://github.com/castorini/daam) + create heatmap visualizations of which parts of the prompt influenced which parts of the image + available in scripts for sdxl text-to-image workflows +- **Offloading** + - improve offloading for pipelines with multiple stages such as *wan-2.2-14b* + - add timers to measure onload/offload times during generate + - experimental offloading using `torch.streams` + enable in settings -> model offloading + - new feature to specify which models types not to offload + in *settings -> model offloading -> model types not to offload* +- **UI** + - **connection monitor** + main logo in top-left corner now indicates server connection status and hovering over it shows connection details + - separate guidance and detail sections + - networks ability to filter lora by base model version +- **Other** + - server will note when restart is recommended due to package updates + - **interrrupt** will now show last known preview image + *keep incomplete* setting is now *save interrupted* + - **logging** enable `debug`, `docs` and `api-docs` by default + - **logging** add detailed ram/vram utilization info to log + logging frequency can be specified using `--monitor x` command line param, where x is number of seconds + - **ipex** simplify internal implementation + - refactor to use new libraries + - styles and wildcards now use same seed as main generate for reproducible results + - **api** new endpoint POST `/sdapi/v1/civitai` to trigger civitai models metadata update + accepts optional `page` parameter to search specific networks page + - **reference models** additional example images, thanks @liutyi + - **reference models** add model size and release date, thanks @alerikaisattera + - **video** support for configurable multi-stage models such as WAN-2.2-14B + - **video** new LTX model selection + - replace `pynvml` with `nvidia-ml-py` for gpu monitoring + - update **loopback** script with radon seed option, thanks @rabanti + - **vae** slicing enable for lowvram/medvram, tiling for lowvram, both disabled otherwise + - **attention** remove split-attention and add explicitly attention slicing enable/disable option + enable in *settings -> compute settings* + can be combined with sdp, enabling may improve stability when used on iGPU or shared memory systems +- **Experimental** + - `new` command line flag enables new `pydantic` and `albumentations` packages + - **modular pipelines**: enable in *settings -> model options* + only compatible with some pipelines, invalidates preview generation + - **modular guiders**: automatically used for compatible pipelines when *modular pipelines* is enabled + allows for using many different guidance methods: + *CFG, CFGZero, PAG, APG, SLG, SEG, TCFG, FDG* +- **Fixes** + - **Microsoft Florence 2** both base and large variants + *note* this will trigger download of the new variant of the model, feel free to delete older variant in `huggingface` folder + - **MiaoshouAI PromptGen** 1.5/2.0 in both base and large variants + - ui: fix image metadata display when switching selected image in control tab + - framepack: add explicit hf-login before framepack load + - framepack: patch solver for unsupported gpus + - benchmark: remove forced sampler from system info benchmark + - xyz-grid: fix xyz grid with random seeds + - reference: fix download for sd15/sdxl reference models + - fix checks in init/mask image decode + ## Update for 2025-09-15 ### Highlights for 2025-09-15 @@ -82,6 +175,13 @@ And check out new **history** tab in the right panel, it now shows visualization *experimental*: requires new pydantic package which *may* break other things, to enable start sdnext with `--new` flag *note*: this is model quantization only, no support for tensorRT inference yet - **Other** + - **LoRA** allow specifying module to apply lora on + *example*: `` would apply lora *only* on unet regardless of lora content + this is particularly useful when you have multiple loras and you want to apply them on different parts of the model + *example*: `` and `` + *note*: `low` is shorthand for `module=transformer_2` and `high` is shortcut for `module=transformer` + - **Detailer** allow manually setting processing resolution + *note*: this does not impact the actual image resolution, only the resolution at which detailer internally operates - refactor reuse-seed and add functionality to all tabs - refactor modernui js codebase - move zluda flash attenion to *Triton Flash attention* option @@ -96,8 +196,6 @@ And check out new **history** tab in the right panel, it now shows visualization - add deprecation warning for `python==3.9` - allow setting denoise strength to 0 in control/img2img this allows to run workflows which only refine or detail existing image without changing it - - **Detailer** allow manually setting processing resolution - *note*: this does not impact the actual image resolution, only the resolution at which detailer internally operates - **Fixes** - normalize path hanlding when deleting images - unified compile upscalers diff --git a/README.md b/README.md index e37f77479..f57feb916 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ ![Last update](https://img.shields.io/github/last-commit/vladmandic/sdnext?svg=true) ![License](https://img.shields.io/github/license/vladmandic/sdnext?svg=true) [![Discord](https://img.shields.io/discord/1101998836328697867?logo=Discord&svg=true)](https://discord.gg/VjvR2tabEX) +[![DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/vladmandic/sdnext) [![Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/vladmandic) [Docs](https://vladmandic.github.io/sdnext-docs/) | [Wiki](https://github.com/vladmandic/sdnext/wiki) | [Discord](https://discord.gg/VjvR2tabEX) | [Changelog](CHANGELOG.md) diff --git a/TODO.md b/TODO.md index dce3ad49e..1a5475c7d 100644 --- a/TODO.md +++ b/TODO.md @@ -13,13 +13,9 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - Video Core: API - Video LTX: TeaCache and others, API, Conditioning preprocess Video: LTX API -### Blocked items - -- Upgrade: `albumentations` - - blocked by `insightface` - ### Under Consideration +- [Inf-DiT](https://github.com/zai-org/Inf-DiT) - [X-Omni](https://github.com/X-Omni-Team/X-Omni/blob/main/README.md) - [DiffSynth Studio](https://github.com/modelscope/DiffSynth-Studio) - [IPAdapter negative guidance](https://github.com/huggingface/diffusers/discussions/7167) @@ -40,14 +36,18 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ### New models +- [Kandinsky 5](https://github.com/ai-forever/Kandinsky-5) +- [Bytedance Lynx](https://github.com/bytedance/lynx) - [HunyuanImage](https://huggingface.co/tencent/HunyuanImage-2.1) +- [Phantom HuMo](https://github.com/Phantom-video/Phantom) - [Lumina-DiMOO](https://huggingface.co/Alpha-VLLM/Lumina-DiMOO) -- [Wan2.2 Fun](https://huggingface.co/collections/alibaba-pai/wan22-fun-68958eabec343b948f1225c5) (includes VACE, Control, etc) +- [Wan2.2-Animate-14B](https://huggingface.co/Wan-AI/Wan2.2-Animate-14B) - [Magi](https://github.com/SandAI-org/MAGI-1)(https://github.com/huggingface/diffusers/pull/11713) - [SEVA](https://github.com/huggingface/diffusers/pull/11440) - [Ming](https://github.com/inclusionAI/Ming) - [Liquid](https://github.com/FoundationVision/Liquid) - [Step1X](https://github.com/stepfun-ai/Step1X-Edit) +- [LucyEdit](https://github.com/huggingface/diffusers/pull/12340) - [SD3 UltraEdit](https://github.com/HaozheZhao/UltraEdit) - [WAN2GP](https://github.com/deepbeepmeep/Wan2GP) - [SelfForcing](https://github.com/guandeh17/Self-Forcing) diff --git a/cli/localize.js b/cli/localize.js index da870d455..12d289867 100755 --- a/cli/localize.js +++ b/cli/localize.js @@ -7,9 +7,10 @@ const { GoogleGenerativeAI } = require('@google/generative-ai'); const api_key = process.env.GOOGLE_AI_API_KEY; const model = 'gemini-2.5-flash'; -const prompt = ` -// eslint-disable-next-line max-len -Translate attached JSON from English to {language} using following rules: fields id, label and reload should be preserved from original, field localized should be a translated version of field label and field hint should be translated in-place. if field is less than 3 characters, do not translate it and keep it as is. Every JSON entry should have id, label, localized, reload and hint fields. Output should be pure JSON without any additional text. To better match translation, context of the text is related to Stable Diffusion and topic of Generative AI.`; +const prompt = `Translate attached JSON from English to {language} using following rules: fields id, label and reload should be preserved from original, field localized should be a translated version of field label and field hint should be translated in-place. +if field is less than 3 characters, do not translate it and keep it as is. +Every JSON entry should have id, label, localized, reload and hint fields. +Output should be pure JSON without any additional text. To better match translation, context of the text is related to Stable Diffusion and topic of Generative AI.`; const languages = { hr: 'Croatian', de: 'German', diff --git a/configs/Dockerfile.cuda b/configs/Dockerfile.cuda index 4e5a496e1..b53a8872f 100644 --- a/configs/Dockerfile.cuda +++ b/configs/Dockerfile.cuda @@ -41,11 +41,11 @@ ENV SD_DOCKER=true # tcmalloc is not required but it is highly recommended ENV LD_PRELOAD=libtcmalloc.so.4 # sdnext will run all necessary pip install ops and then exit -RUN ["python", "/app/launch.py", "--debug", "--uv", "--use-cuda", "--log", "sdnext.log", "--test", "--optional"] +RUN ["python", "/app/launch.py", "--uv", "--use-cuda", "--log", "sdnext.log", "--test", "--optional"] # preinstall additional packages to avoid installation during runtime # actually run sdnext -CMD ["python", "launch.py", "--debug", "--skip-all", "--listen", "--quick", "--api-log", "--log", "sdnext.log"] +CMD ["python", "launch.py", "--listen", "--quick", "--log", "sdnext.log"] # expose port EXPOSE 7860 diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index 615d2f810..19a1b1b72 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit 615d2f8103fb61779037bd81523906457fbf7277 +Subproject commit 19a1b1b722b768b821f67831a6d3ba83847bcb61 diff --git a/extensions-builtin/sd-webui-agent-scheduler b/extensions-builtin/sd-webui-agent-scheduler deleted file mode 160000 index a33753321..000000000 --- a/extensions-builtin/sd-webui-agent-scheduler +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a33753321b914c6122df96d1dc0b5117d38af680 diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 430c8140c..e56b4f695 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 430c8140c0cdda5e82a80f44ee6961e0d803e516 +Subproject commit e56b4f69549e6654c953346082ff3947256c9636 diff --git a/html/locale_en.json b/html/locale_en.json index 8113a811b..056637d2b 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -32,7 +32,6 @@ {"id":"","label":"","localized":"","reload":"","hint":"Sort by time, descending"} ], "main": [ - {"id":"","label":"SD.Next","localized":"","reload":"","hint":"SD.Next
All-in-one WebUI for AI generative image and video creation"}, {"id":"","label":"Prompt","localized":"","reload":"","hint":"Describe image you want to generate"}, {"id":"","label":"Start","localized":"","reload":"","hint":"Start"}, {"id":"","label":"End","localized":"","reload":"","hint":"End"}, diff --git a/html/previews.json b/html/previews.json index 6f77db164..07af08547 100644 --- a/html/previews.json +++ b/html/previews.json @@ -1,6 +1,6 @@ { "sd-v21-512-ema": "models/Reference/stabilityai--stable-diffusion-2-1-base.jpg", - "stabilityai--stable-diffusion-xl-base-1.0": "models/Reference/sd_xl_base_1.0.jpg", + "stabilityai--stable-diffusion-xl-base-1.0": "models/Reference/stabilityai--stable-diffusion-xl-base-1.0.jpg", "stabilityai--stable-diffusion-3-medium-diffusers": "models/Reference/stabilityai--stable-diffusion-3.jpg", "stabilityai--stable-diffusion-3.5-medium": "models/Reference/stabilityai--stable-diffusion-3_5-medium.jpg", "stabilityai--stable-diffusion-3.5-large": "models/Reference/stabilityai--stable-diffusion-3_5-large.jpg", @@ -20,10 +20,11 @@ "HiDream-I1-Full": "models/Reference/HiDream-I1 Full", "lodestones--Chroma1-Base": "models/Reference/lodestones--Chroma-Base.jpg", "lodestones--Chroma1-HD": "models/Reference/lodestones--Chroma-HD.jpg", - "chroma-unlocked-v50": "models/Reference/lodestones Chroma Unlocked HD", - "chroma-unlocked-v50-annealed": "models/Reference/lodestones Chroma Unlocked HD", + "chroma-unlocked-v50": "models/Reference/lodestones--Chroma-detail.jpg", + "chroma-unlocked-v50-annealed": "models/Reference/lodestones--Chroma-annealed.jpg", "vladmandic--Qwen-Lightning": "models/Reference/Qwen-Lightning.jpg", "vladmandic--Qwen-Lightning-Edit": "models/Reference/Qwen-Lightning.jpg", - "Wan-AI--Wan2.2-T2V-A14B-Diffusers": "models/Reference/Wan2.2-T2V-A14B.jpg", - "Wan-AI--Wan2.1-T2V-14B-Diffusers": "models/Reference/Wan-AI--Wan2.1.jpg" + "Wan-AI--Wan2.2-T2V-A14B-Diffusers": "models/Reference/Wan-AI--Wan2.2-T2V-A14B-Diffusers.jpg", + "Wan-AI--Wan2.1-T2V-14B-Diffusers": "models/Reference/Wan-AI--Wan2.1-T2V-14B-Diffusers.jpg", + "linoyts--Wan2.2-VACE-Fun-14B-diffusers": "models/Reference/linoyts--Wan2.2-VACE-Fun-14B-diffusers.jpg" } diff --git a/html/reference.json b/html/reference.json index ee0ccf103..8c5901381 100644 --- a/html/reference.json +++ b/html/reference.json @@ -1,26 +1,26 @@ { "Tempest-by-Vlad XL": { "path": "tempestByVlad_baseV01.safetensors@https://civitai.com/api/download/models/1301775", - "preview": "tempest-by-vlad-base.jpg", + "preview": "tempestByVlad_baseV01.jpg", "desc": "Flexible SDXL model with custom encoder and finetuned for larger landscape resolutions with high details and high contrast.", "extras": "" }, "Tempest-by-Vlad XL Hyper": { "path": "tempestByVlad_hyperV01.safetensors@https://civitai.com/api/download/models/1343512", - "preview": "tempest-by-vlad-hyper.jpg", + "preview": "tempestByVlad_hyperV01.jpg", "desc": "Custom distilled variant with goal to get as-normal-as-possible model that works with low steps and guidance-free", "extras": "" }, "Juggernaut XL XI": { "path": "juggernautXL_juggXIByRundiffusion.safetensors@https://civitai.com/api/download/models/782002", - "preview": "juggernautXL_v9Rundiffusionphoto2.jpg", + "preview": "juggernautXL_juggXIByRundiffusion.jpg", "desc": "Showcase finetuned model based on Stable diffusion XL", "extras": "sampler: DEIS, steps: 20, cfg_scale: 6.0" }, "Juggernaut XL XI Lightning": { "path": "juggernautXL_juggXILightningByRD.safetensors@https://civitai.com/api/download/models/920957", - "preview": "juggernautXL_v9Rdphoto2Lightning.jpg", + "preview": "juggernautXL_juggXILightningByRD.jpg", "desc": "Showcase finetuned model based on Stable diffusion XL", "extras": "sampler: DPM SDE, steps: 6, cfg_scale: 2.0" }, @@ -37,7 +37,9 @@ "path": "v1-5-pruned-fp16-emaonly.safetensors@https://huggingface.co/Aptronym/SDNext/resolve/main/Reference/v1-5-pruned-fp16-emaonly.safetensors?download=true", "preview": "v1-5-pruned-fp16-emaonly.jpg", "desc": "Stable Diffusion 1.5 is the base model all other 1.5 checkpoint were trained from. It's a latent text-to-image diffusion model capable of generating photo-realistic images given any text input. The Stable-Diffusion-v1-5 checkpoint was initialized with the weights of the Stable-Diffusion-v1-2 checkpoint and subsequently fine-tuned on 595k steps at resolution 512x512.", - "extras": "width: 512, height: 512, sampler: DEIS, steps: 20, cfg_scale: 6.0" + "extras": "width: 512, height: 512, sampler: DEIS, steps: 20, cfg_scale: 6.0", + "size": 2.28, + "date": "2022 October" }, "StabilityAI StableDiffusion 2.1": { "path": "huggingface/stabilityai/stable-diffusion-2-1-base", @@ -45,7 +47,9 @@ "skip": true, "variant": "fp16", "desc": "This stable-diffusion-2-1-base model fine-tunes stable-diffusion-2-base (512-base-ema.ckpt) with 220k extra steps taken", - "extras": "width: 512, height: 512, sampler: DEIS, steps: 20, cfg_scale: 6.0" + "extras": "width: 512, height: 512, sampler: DEIS, steps: 20, cfg_scale: 6.0", + "size": 2.58, + "date": "2022 December" }, "StabilityAI StableDiffusion 2.1 V": { "path": "huggingface/stabilityai/stable-diffusion-2-1", @@ -55,11 +59,23 @@ "desc": "This stable-diffusion-2 model is resumed from stable-diffusion-2-base (512-base-ema.ckpt) and trained for 150k steps using a v-objective on the same dataset. Resumed for another 140k steps on 768x768 images", "extras": "width: 768, height: 768, sampler: DEIS, steps: 20, cfg_scale: 6.0" }, - "StabilityAI StableDiffusion XL 1.0 Base": { - "path": "sd_xl_base_1.0.safetensors@https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0.safetensors?download=true", - "preview": "sd_xl_base_1.0.jpg", - "desc": "Stable Diffusion XL (SDXL) is the latest AI image generation model that is tailored towards more photorealistic outputs with more detailed imagery and composition compared to previous SD models, including SD 2.1. It can make realistic faces, legible text within the images, and better image composition, all while using shorter and simpler prompts at a greatly increased base resolution of 1024x1024. Just like its predecessors, SDXL has the ability to generate image variations using image-to-image prompting, inpainting (reimagining of the selected parts of an image), and outpainting (creating new parts that lie outside the image borders).", - "extras": "sampler: DEIS, steps: 20, cfg_scale: 6.0" + "StabilityAI StableDiffusion XL": { + "path": "stabilityai/stable-diffusion-xl-base-1.0", + "preview": "stabilityai--stable-diffusion-xl-base-1.0.jpg", + "desc": "Stable Diffusion XL (SDXL) is AI image generation model that is tailored towards more photorealistic outputs with more detailed imagery and composition compared to previous SD models, including SD 2.1. It can make realistic faces and better image composition, all while using shorter and simpler prompts at a greatly increased base resolution of 1024x1024. Just like its predecessors, SDXL has the ability to generate image variations using image-to-image prompting, inpainting (reimagining of the selected parts of an image), and outpainting (creating new parts that lie outside the image borders).", + "skip": true, + "variant": "fp16", + "extras": "", + "size": 6.94, + "date": "2023 July" + }, + "StabilityAI StableDiffusion XL Turbo": { + "path": "stabilityai/sdxl-turbo", + "preview": "stabilityai--sdxl-turbo.jpg", + "desc": "SDXL-Turbo is a fast generative text-to-image model that can synthesize photorealistic images from a text prompt in a 1-4 steps.", + "skip": true, + "variant": "fp16", + "extras": "steps: 4, cfg_scale: 0.0" }, "StabilityAI Stable Cascade": { "path": "huggingface/stabilityai/stable-cascade", @@ -67,7 +83,9 @@ "variant": "bf16", "desc": "Stable Cascade is a diffusion model built upon the Würstchen architecture and its main difference to other models like Stable Diffusion is that it is working at a much smaller latent space. Why is this important? The smaller the latent space, the faster you can run inference and the cheaper the training becomes. How small is the latent space? Stable Diffusion uses a compression factor of 8, resulting in a 1024x1024 image being encoded to 128x128. Stable Cascade achieves a compression factor of 42, meaning that it is possible to encode a 1024x1024 image to 24x24, while maintaining crisp reconstructions. The text-conditional model is then trained in the highly compressed latent space. Previous versions of this architecture, achieved a 16x cost reduction over Stable Diffusion 1.5", "preview": "stabilityai--stable-cascade.jpg", - "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0" + "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0", + "size": 11.82, + "date": "2024 February" }, "StabilityAI Stable Cascade Lite": { "path": "huggingface/stabilityai/stable-cascade-lite", @@ -75,7 +93,9 @@ "variant": "bf16", "desc": "Stable Cascade is a diffusion model built upon the Würstchen architecture and its main difference to other models like Stable Diffusion is that it is working at a much smaller latent space. Why is this important? The smaller the latent space, the faster you can run inference and the cheaper the training becomes. How small is the latent space? Stable Diffusion uses a compression factor of 8, resulting in a 1024x1024 image being encoded to 128x128. Stable Cascade achieves a compression factor of 42, meaning that it is possible to encode a 1024x1024 image to 24x24, while maintaining crisp reconstructions. The text-conditional model is then trained in the highly compressed latent space. Previous versions of this architecture, achieved a 16x cost reduction over Stable Diffusion 1.5", "preview": "stabilityai--stable-cascade-lite.jpg", - "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0" + "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0", + "size": 4.97, + "date": "2024 February" }, "StabilityAI Stable Diffusion 3 Medium": { "path": "stabilityai/stable-diffusion-3-medium-diffusers", @@ -83,7 +103,9 @@ "variant": "fp16", "desc": "Stable Diffusion 3 Medium is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features greatly improved performance in image quality, typography, complex prompt understanding, and resource-efficiency", "preview": "stabilityai--stable-diffusion-3.jpg", - "extras": "sampler: Default, cfg_scale: 7.0" + "extras": "sampler: Default, cfg_scale: 7.0", + "size": 15.14, + "date": "2024 June" }, "StabilityAI Stable Diffusion 3.5 Medium": { "path": "stabilityai/stable-diffusion-3.5-medium", @@ -91,7 +113,9 @@ "variant": "fp16", "desc": "Stable Diffusion 3.5 Medium is a Multimodal Diffusion Transformer with improvements (MMDiT-X) text-to-image model that features improved performance in image quality, typography, complex prompt understanding, and resource-efficiency.", "preview": "stabilityai--stable-diffusion-3_5-medium.jpg", - "extras": "sampler: Default, cfg_scale: 7.0" + "extras": "sampler: Default, cfg_scale: 7.0", + "size": 15.89, + "date": "2024 October" }, "StabilityAI Stable Diffusion 3.5 Large": { "path": "stabilityai/stable-diffusion-3.5-large", @@ -99,7 +123,9 @@ "variant": "fp16", "desc": "Stable Diffusion 3.5 Large is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features improved performance in image quality, typography, complex prompt understanding, and resource-efficiency.", "preview": "stabilityai--stable-diffusion-3_5-large.jpg", - "extras": "sampler: Default, cfg_scale: 7.0" + "extras": "sampler: Default, cfg_scale: 7.0", + "size": 26.98, + "date": "2024 October" }, "StabilityAI Stable Diffusion 3.5 Turbo": { "path": "stabilityai/stable-diffusion-3.5-large-turbo", @@ -115,107 +141,153 @@ "preview": "black-forest-labs--FLUX.1-dev.jpg", "desc": "FLUX.1 models are based on a hybrid architecture of multimodal and parallel diffusion transformer blocks, scaled to 12B parameters and builing on flow matching", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 32.93, + "date": "2024 August" }, "Black Forest Labs FLUX.1 Schnell": { "path": "black-forest-labs/FLUX.1-schnell", "preview": "black-forest-labs--FLUX.1-schnell.jpg", "desc": "FLUX.1 models are based on a hybrid architecture of multimodal and parallel diffusion transformer blocks, scaled to 12B parameters and builing on flow matching. Trained using latent adversarial diffusion distillation, FLUX.1 [schnell] can generate high-quality images in only 1 to 4 steps", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 32.93, + "date": "2024 August" }, "Black Forest Labs FLUX.1 Kontext Dev": { "path": "black-forest-labs/FLUX.1-Kontext-dev", "preview": "black-forest-labs--FLUX.1-Kontext-dev.jpg", "desc": "FLUX.1 Kontext [dev] is a 12 billion parameter rectified flow transformer capable of editing images based on text instructions.", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 32.93, + "date": "2025 June" }, "Black Forest Labs FLUX.1 Krea Dev": { "path": "black-forest-labs/FLUX.1-Krea-dev", "preview": "black-forest-labs--FLUX.1-Krea-dev.jpg", "desc": "FLUX.1 Krea [dev] is a 12 billion parameter rectified flow transformer capable of generating images from text descriptions.", "skip": true, + "extras": "sampler: Default, cfg_scale: 4.5", + "size": 32.93, + "date": "2025 July" + }, + "Tencent FLUX.1 Dev SRPO": { + "path": "vladmandic/flux.1-dev-SRPO", + "preview": "vladmandic--flux.1-dev-SRPO.jpg", + "desc": "FLUX.1 Dev SRPO is Tencent trained with specific technique: Directly Aligning the Full Diffusion Trajectory with Fine-Grained Human Preference", + "skip": true, "extras": "sampler: Default, cfg_scale: 4.5" }, - "lodestones Chroma1 HD": { - "path": "lodestones/Chroma1-HD", - "preview": "lodestones--Chroma-HD.jpg", - "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. This is the high-res fine-tune of the Chroma1-Base at a 1024x1024 resolution.", - "skip": true, - "extras": "" - }, - "lodestones Chroma1 Base": { - "path": "lodestones/Chroma1-Base", - "preview": "lodestones--Chroma-Base.jpg", - "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. This is the core 512x512 model. It's a solid, all-around foundation for pretty much any creative project.", - "skip": true, - "extras": "" - }, - "lodestones Chroma1 Flash": { - "path": "lodestones/Chroma1-Flash", - "preview": "lodestones--Chroma-flash.jpg", - "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. A fine-tuned version of the Chroma1-Base made to find the best way to make these flow matching models faster.", - "skip": true, - "extras": "" - }, - "lodestones Chroma1 v50 Preview Annealed": { - "path": "vladmandic/chroma-unlocked-v50-annealed", - "preview": "lodestones--Chroma-annealed.jpg", - "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. Re-tweaked variant with extra noise added.", - "skip": true, - "extras": "" - }, - "lodestones Chroma1 v48 Preview": { - "path": "vladmandic/chroma-unlocked-v48", - "preview": "lodestones--Chroma.jpg", - "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. Last raw version of Chroma before final finetuning.", - "skip": true, - "extras": "" - }, - "lodestones Chroma1 v48 Preview Calibrated": { - "path": "vladmandic/chroma-unlocked-v48-detail-calibrated", - "preview": "lodestones--Chroma-detail.jpg", - "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. Last raw version of Chroma before final finetuning but with some detail calibration.", - "skip": true, - "extras": "" - }, - "Qwen-Image": { "path": "Qwen/Qwen-Image", "preview": "Qwen--Qwen-Image.jpg", "desc": " Qwen-Image, an image generation foundation model in the Qwen series that achieves significant advances in complex text rendering and precise image editing.", "skip": true, - "extras": "" + "extras": "", + "size": 56.1, + "date": "2025 August" }, "Qwen-Image-Edit": { "path": "Qwen/Qwen-Image-Edit", "preview": "Qwen--Qwen-Image-Edit.jpg", "desc": "Qwen-Image-Edit, the image editing version of Qwen-Image. Built upon our 20B Qwen-Image model, Qwen-Image-Edit successfully extends Qwen-Image’s unique text rendering capabilities to image editing tasks, enabling precise text editing.", "skip": true, - "extras": "" + "extras": "", + "size": 56.1, + "date": "2025 August" + }, + "Qwen-Image-Edit-2509": { + "path": "Qwen/Qwen-Image-Edit-2509", + "preview": "Qwen--Qwen-Image-Edit-2509.jpg", + "desc": "Qwen-Image-Edit, the image editing version of Qwen-Image. Built upon our 20B Qwen-Image model, Qwen-Image-Edit successfully extends Qwen-Image’s unique text rendering capabilities to image editing tasks, enabling precise text editing.", + "skip": true, + "extras": "", + "size": 56.1, + "date": "2025 September" }, "Qwen-Image-Lightning": { "path": "vladmandic/Qwen-Lightning", "preview": "vladmandic--Qwen-Lightning.jpg", "desc": "Qwen-Lightning is step-distilled from Qwen-Image to allow for generation in 8 steps.", "skip": true, - "extras": "steps: 8" + "extras": "steps: 8", + "size": 56.1, + "date": "2025 August" }, "Qwen-Image-Distill": { "path": "SahilCarterr/Qwen-Image-Distill-Full", "preview": "SahilCarterr--Qwen-Image-Distill-Full.jpg", "desc": "Qwen-Image-Distill is a distilled and accelerated version of Qwen-Image by DiffSynth-Studio.", "skip": true, - "extras": "steps: 15" + "extras": "steps: 15", + "size": 56.1, + "date": "2025 August" }, "Qwen-Image-Lightning-Edit": { "path": "vladmandic/Qwen-Lightning-Edit", "preview": "vladmandic--Qwen-Lightning-Edit.jpg", "desc": " Qwen-Lightning-Edit is step-distilled from Qwen-Image-Edit to allow for generation in 8 steps.", "skip": true, - "extras": "steps: 8" + "extras": "steps: 8", + "size": 56.1, + "date": "2025 August" + }, + + "lodestones Chroma1 HD": { + "path": "lodestones/Chroma1-HD", + "preview": "lodestones--Chroma-HD.jpg", + "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. This is the high-res fine-tune of the Chroma1-Base at a 1024x1024 resolution.", + "skip": true, + "extras": "", + "size": 26.84, + "date": "2025 July" + }, + "lodestones Chroma1 Base": { + "path": "lodestones/Chroma1-Base", + "preview": "lodestones--Chroma-Base.jpg", + "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. This is the core 512x512 model. It's a solid, all-around foundation for pretty much any creative project.", + "skip": true, + "extras": "", + "size": 26.84, + "date": "2025 July" + }, + "lodestones Chroma1 Flash": { + "path": "lodestones/Chroma1-Flash", + "preview": "lodestones--Chroma-flash.jpg", + "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. A fine-tuned version of the Chroma1-Base made to find the best way to make these flow matching models faster.", + "skip": true, + "extras": "", + "size": 26.84, + "date": "2025 July" + }, + "lodestones Chroma1 v50 Preview Annealed": { + "path": "vladmandic/chroma-unlocked-v50-annealed", + "preview": "lodestones--Chroma-annealed.jpg", + "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. Re-tweaked variant with extra noise added.", + "skip": true, + "extras": "", + "size": 26.84, + "date": "2025 July" + }, + "lodestones Chroma1 v48 Preview": { + "path": "vladmandic/chroma-unlocked-v48", + "preview": "lodestones--Chroma.jpg", + "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. Last raw version of Chroma before final finetuning.", + "skip": true, + "extras": "", + "size": 26.84, + "date": "2025 July" + }, + "lodestones Chroma1 v48 Preview Calibrated": { + "path": "vladmandic/chroma-unlocked-v48-detail-calibrated", + "preview": "lodestones--Chroma-detail.jpg", + "desc": "Chroma is a 8.9B parameter model based on FLUX.1-schnell. It’s fully Apache 2.0 licensed, ensuring that anyone can use, modify, and build on top of it—no corporate gatekeeping. Last raw version of Chroma before final finetuning but with some detail calibration.", + "skip": true, + "extras": "", + "size": 26.84, + "date": "2025 July" }, "Ostris Flex.2 Preview": { @@ -223,14 +295,18 @@ "preview": "ostris--Flex.2-preview.jpg", "desc": "Open Source 8B parameter Text to Image Diffusion Model with universal control and inpainting support built in. Early access preview release. The next version of Flex.1-alpha", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 25.65, + "date": "2025 April" }, "Ostris Flex.1 Alpha": { "path": "ostris/Flex.1-alpha", "preview": "ostris--Flex.1-alpha.jpg", "desc": "Flex.1 alpha is a pre-trained base 8 billion parameter rectified flow transformer capable of generating images from text descriptions. It has a similar architecture to FLUX.1-dev, but with fewer double transformer blocks (8 vs 19)", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 25.65, + "date": "2025 January" }, "Wan-AI Wan2.1 1.3B": { @@ -238,14 +314,18 @@ "preview": "Wan-AI--Wan2.1-T2V-1.3B-Diffusers.jpg", "desc": "Wan is an advanced and powerful visual generation model developed by Tongyi Lab of Alibaba Group. It can generate videos based on text, images, and other control signals. The Wan2.1 series models are now fully open-source.", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 27.72, + "date": "2025 February" }, "Wan-AI Wan2.1 14B": { "path": "Wan-AI/Wan2.1-T2V-14B-Diffusers", "preview": "Wan-AI--Wan2.1-T2V-14B-Diffusers.jpg", "desc": "Wan is an advanced and powerful visual generation model developed by Tongyi Lab of Alibaba Group. It can generate videos based on text, images, and other control signals. The Wan2.1 series models are now fully open-source.", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 78.52, + "date": "2025 February" }, "Wan-AI Wan2.2 5B": { "path": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", @@ -268,27 +348,40 @@ "skip": true, "extras": "sampler: Default" }, + "Wan-AI Wan2.2 14B VACE": { + "path": "linoyts/Wan2.2-VACE-Fun-14B-diffusers", + "preview": "linoyts--Wan2.2-VACE-Fun-14B-diffusers.jpg", + "desc": "Wan2.2, offering more powerful capabilities, better performance, and superior visual quality. With Wan2.2, we have focused on incorporating the following technical innovations: MoE Architecture, Data Scalling, Cinematic Aesthetics, Efficient High-Definition Hybrid", + "skip": true, + "extras": "sampler: Default" + }, "Freepik F-Lite": { "path": "Freepik/F-Lite", "preview": "Freepik--F-Lite.jpg", "desc": "F Lite is a 10B parameter diffusion model created by Freepik and Fal, trained exclusively on copyright-safe and SFW content. The model was trained on Freepik's internal dataset comprising approximately 80 million copyright-safe images, making it the first publicly available model of this scale trained exclusively on legally compliant and SFW content.", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 19.81, + "date": "2025 May" }, "Freepik F-Lite Texture": { "path": "Freepik/F-Lite-Texture", "preview": "Freepik--F-Lite-Texture.jpg", "desc": "F Lite is a 10B parameter diffusion model created by Freepik and Fal, trained exclusively on copyright-safe and SFW content. The model was trained on Freepik's internal dataset comprising approximately 80 million copyright-safe images, making it the first publicly available model of this scale trained exclusively on legally compliant and SFW content.", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 19.81, + "date": "2025 May" }, "Freepik F-Lite 7B": { "path": "Freepik/F-Lite-7B", "preview": "Freepik--F-Lite-7B.jpg", "desc": "F Lite is a 10B parameter diffusion model created by Freepik and Fal, trained exclusively on copyright-safe and SFW content. The model was trained on Freepik's internal dataset comprising approximately 80 million copyright-safe images, making it the first publicly available model of this scale trained exclusively on legally compliant and SFW content.", "skip": true, - "extras": "sampler: Default, cfg_scale: 3.5" + "extras": "sampler: Default, cfg_scale: 3.5", + "size": 13.89, + "date": "2025 May" }, "SDXS DreamShaper 512": { @@ -309,13 +402,17 @@ "path": "Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers", "desc": "Sana is an efficient model with scaling of training-time and inference time techniques. SANA-1.5 delivers: efficient model growth from 1.6B Sana-1.0 model to 4.8B, achieving similar or better performance than training from scratch and saving 60% training cost; efficient model depth pruning, slimming any model size as you want; powerful VLM selection based inference scaling, smaller model+inference scaling > larger model.", "preview": "Efficient-Large-Model--SANA1.5_1.6B_1024px_diffusers.jpg", - "skip": true + "skip": true, + "size": 9.49, + "date": "2025 March" }, "NVLabs Sana 1.5 4.8B 1k": { "path": "Efficient-Large-Model/SANA1.5_4.8B_1024px_diffusers", "desc": "Sana is an efficient model with scaling of training-time and inference time techniques. SANA-1.5 delivers: efficient model growth from 1.6B Sana-1.0 model to 4.8B, achieving similar or better performance than training from scratch and saving 60% training cost; efficient model depth pruning, slimming any model size as you want; powerful VLM selection based inference scaling, smaller model+inference scaling > larger model.", "preview": "Efficient-Large-Model--SANA1.5_4.8B_1024px_diffusers.jpg", - "skip": true + "skip": true, + "size": 15.58, + "date": "2025 March" }, "NVLabs Sana 1.5 1.6B 1k Sprint": { "path": "Efficient-Large-Model/Sana_Sprint_1.6B_1024px_diffusers", @@ -327,63 +424,83 @@ "path": "Efficient-Large-Model/Sana_1600M_4Kpx_BF16_diffusers", "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", "preview": "Efficient-Large-Model--Sana_1600M_4Kpx_BF16_diffusers.jpg", - "skip": true + "skip": true, + "size": 12.63, + "date": "2024 November" }, "NVLabs Sana 1.0 1.6B 2k": { "path": "Efficient-Large-Model/Sana_1600M_2Kpx_BF16_diffusers", "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", "preview": "Efficient-Large-Model--Sana_1600M_2Kpx_BF16_diffusers.jpg", - "skip": true + "skip": true, + "size": 12.63, + "date": "2024 November" }, "NVLabs Sana 1.0 1.6B 1k": { "path": "Efficient-Large-Model/Sana_1600M_1024px_diffusers", "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", "preview": "Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg", - "skip": true + "skip": true, + "size": 12.63, + "date": "2024 November" }, "NVLabs Sana 1.0 0.6B 0.5k": { "path": "Efficient-Large-Model/Sana_600M_512px_diffusers", "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", "preview": "Efficient-Large-Model--Sana_600M_512px_diffusers.jpg", - "skip": true + "skip": true, + "size": 7.51, + "date": "2024 November" }, "nVidia Cosmos-Predict2 T2I 2B": { "path": "nvidia/Cosmos-Predict2-2B-Text2Image", "desc": "Cosmos-Predict2: A family of highly performant pre-trained world foundation models purpose-built for generating physics-aware images, videos and world states for physical AI development.", "preview": "nvidia--Cosmos-Predict2-2B-Text2Image.jpg", - "skip": true + "skip": true, + "size": 13.32, + "date": "2025 June" }, "nVidia Cosmos-Predict2 T2I 14B": { "path": "nvidia/Cosmos-Predict2-14B-Text2Image", "desc": "Cosmos-Predict2: A family of highly performant pre-trained world foundation models purpose-built for generating physics-aware images, videos and world states for physical AI development.", "preview": "nvidia--Cosmos-Predict2-14B-Text2Image.jpg", - "skip": true + "skip": true, + "size": 37.36, + "date": "2025 June" }, "VectorSpaceLab OmniGen v1": { "path": "Shitao/OmniGen-v1-diffusers", "desc": "OmniGen is a unified image generation model that can generate a wide range of images from multi-modal prompts. It is designed to be simple, flexible and easy to use.", "preview": "Shitao--OmniGen-v1.jpg", - "skip": true + "skip": true, + "size": 15.47, + "date": "2024 October" }, "VectorSpaceLab OmniGen v2": { "path": "OmniGen2/OmniGen2", "desc": "OmniGen2 is a powerful and efficient unified multimodal model. Unlike OmniGen v1, OmniGen2 features two distinct decoding pathways for text and image modalities, utilizing unshared parameters and a decoupled image tokenizer.", "preview": "OmniGen2--OmniGen2.jpg", - "skip": true + "skip": true, + "size": 30.5, + "date": "2025 June" }, "AuraFlow 0.3": { "path": "fal/AuraFlow-v0.3", "desc": "AuraFlow v0.3 is the fully open-sourced flow-based text-to-image generation model. The model was trained with more compute compared to the previous version, AuraFlow-v0.2. Compared to AuraFlow-v0.2, the model is fine-tuned on more aesthetic datasets and now supports various aspect ratio, (now width and height up to 1536 pixels).", "preview": "fal--AuraFlow-v0.3.jpg", - "skip": true + "skip": true, + "size": 31.9, + "date": "2024 August" }, "AuraFlow 0.2": { "path": "fal/AuraFlow-v0.2", "desc": "AuraFlow v0.2 is the fully open-sourced largest flow-based text-to-image generation model. The model was trained with more compute compared to the previous version, AuraFlow-v0.1", "preview": "fal--AuraFlow-v0.2.jpg", - "skip": true + "skip": true, + "size": 31.9, + "date": "2024 July" }, "Segmind Vega": { @@ -392,7 +509,9 @@ "desc": "The Segmind-Vega Model is a distilled version of the Stable Diffusion XL (SDXL), offering a remarkable 70% reduction in size and an impressive 100% speedup while retaining high-quality text-to-image generation capabilities. Trained on diverse datasets, including Grit and Midjourney scrape data, it excels at creating a wide range of visual content based on textual prompts. Employing a knowledge distillation strategy, Segmind-Vega leverages the teachings of several expert models, including SDXL, ZavyChromaXL, and JuggernautXL, to combine their strengths and produce compelling visual outputs.", "variant": "fp16", "skip": true, - "extras": "sampler: Default, cfg_scale: 9.0" + "extras": "sampler: Default, cfg_scale: 9.0", + "size": 6.43, + "date": "2023 November" }, "Segmind SSD-1B": { "path": "huggingface/segmind/SSD-1B", @@ -400,13 +519,17 @@ "desc": "The Segmind Stable Diffusion Model (SSD-1B) offers a compact, efficient, and distilled version of the SDXL model. At 50% smaller and 60% faster than Stable Diffusion XL (SDXL), it provides quick and seamless performance without sacrificing image quality.", "variant": "fp16", "skip": true, - "extras": "sampler: Default, cfg_scale: 9.0" + "extras": "sampler: Default, cfg_scale: 9.0", + "size": 8.72, + "date": "2023 October" }, "Segmind Tiny": { "path": "segmind/tiny-sd", "preview": "segmind--tiny-sd.jpg", "desc": "Segmind's Tiny-SD offers a compact, efficient, and distilled version of Realistic Vision 4.0 and is up to 80% faster than SD1.5", - "extras": "width: 512, height: 512, sampler: Default, cfg_scale: 9.0" + "extras": "width: 512, height: 512, sampler: Default, cfg_scale: 9.0", + "size": 1.03, + "date": "2023 July" }, "Segmind SegMoE SD 4x2": { "path": "segmind/SegMoE-SD-4x2-v0", @@ -430,7 +553,9 @@ "path": "PixArt-alpha/PixArt-XL-2-1024-MS", "desc": "PixArt-α is a Transformer-based T2I diffusion model whose image generation quality is competitive with state-of-the-art image generators (e.g., Imagen, SDXL, and even Midjourney), and the training speed markedly surpasses existing large-scale T2I models. Extensive experiments demonstrate that PIXART-α excels in image quality, artistry, and semantic control. It can directly generate 1024px images from text prompts within a single sampling process.", "preview": "PixArt-alpha--PixArt-XL-2-1024-MS.jpg", - "extras": "sampler: Default, cfg_scale: 2.0" + "extras": "sampler: Default, cfg_scale: 2.0", + "size": 21.3, + "date": "2023 November" }, "Pixart-Σ Small": { "path": "huggingface/PixArt-alpha/PixArt-Sigma-XL-2-512-MS", @@ -451,14 +576,18 @@ "desc": "PixArt-Σ, a Diffusion Transformer model (DiT) capable of directly generating images at 4K resolution. PixArt-Σ represents a significant advancement over its predecessor, PixArt-α, offering images of markedly higher fidelity and improved alignment with text prompts.", "preview": "PixArt-alpha--PixArt-Sigma-XL-2-2K-MS.jpg", "skip": true, - "extras": "sampler: Default, cfg_scale: 2.0" + "extras": "sampler: Default, cfg_scale: 2.0", + "size": 21.3, + "date": "2024 April" }, "Tencent HunyuanDiT 1.2": { "path": "Tencent-Hunyuan/HunyuanDiT-v1.2-Diffusers", "desc": "Hunyuan-DiT : A Powerful Multi-Resolution Diffusion Transformer with Fine-Grained Chinese Understanding.", "preview": "Tencent-Hunyuan--HunyuanDiT-v1.2-Diffusers.jpg", - "extras": "sampler: Default, cfg_scale: 2.0" + "extras": "sampler: Default, cfg_scale: 2.0", + "size": 14.09, + "date": "2024 May" }, "Tencent HunyuanDiT 1.2 Distilled": { "path": "Tencent-Hunyuan/HunyuanDiT-v1.2-Diffusers-Distilled", @@ -484,14 +613,18 @@ "desc": "The Lumina-Next-SFT is a Next-DiT model containing 2B parameters and utilizes Gemma-2B as the text encoder, enhanced through high-quality supervised fine-tuning (SFT).", "preview": "Alpha-VLLM--Lumina-Next-SFT-diffusers.jpg", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 8.67, + "date": "2024 June" }, "AlphaVLLM Lumina 2": { "path": "Alpha-VLLM/Lumina-Image-2.0", "desc": "A Unified and Efficient Image Generative Model. Lumina-Image-2.0 is a 2 billion parameter flow-based diffusion transformer capable of generating images from text descriptions.", "preview": "Alpha-VLLM--Lumina-Image-2.0.jpg", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 20.75, + "date": "2025 January" }, "HiDream-I1 Fast": { @@ -499,21 +632,27 @@ "desc": "HiDream-I1 is a new open-source image generative foundation model with 17B parameters that achieves state-of-the-art image generation quality within seconds.", "preview": "HiDream-ai--HiDream-I1-Fast.jpg", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 58.4, + "date": "2025 April" }, "HiDream-I1 Dev": { "path": "HiDream-ai/HiDream-I1-Dev", "desc": "HiDream-I1 is a new open-source image generative foundation model with 17B parameters that achieves state-of-the-art image generation quality within seconds.", "preview": "HiDream-ai--HiDream-I1-Dev.jpg", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 58.4, + "date": "2025 April" }, "HiDream-I1 Full": { "path": "HiDream-ai/HiDream-I1-Full", "desc": "HiDream-I1 is a new open-source image generative foundation model with 17B parameters that achieves state-of-the-art image generation quality within seconds.", "preview": "HiDream-ai--HiDream-I1-Full.jpg", "skip": true, - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 58.4, + "date": "2025 April" }, "HiDream-E1 Full": { "path": "HiDream-ai/HiDream-E1-Full", @@ -522,40 +661,58 @@ "skip": true, "extras": "sampler: Default" }, + "HiDream-E1.1": { + "path": "HiDream-ai/HiDream-E1-1", + "desc": "HiDream-E1 is an image editing model built on HiDream-I1.", + "preview": "HiDream-ai--HiDream-E1-Full.jpg", + "skip": true, + "extras": "sampler: Default", + "experimental": true + }, "Kwai Kolors": { "path": "Kwai-Kolors/Kolors-diffusers", "desc": "Kolors is a large-scale text-to-image generation model based on latent diffusion, developed by the Kuaishou Kolors team. Trained on billions of text-image pairs, Kolors exhibits significant advantages over both open-source and proprietary models in visual quality, complex semantic accuracy, and text rendering for both Chinese and English characters. Furthermore, Kolors supports both Chinese and English inputs", "preview": "Kwai-Kolors--Kolors-diffusers.jpg", "skip": true, - "extras": "width: 1024, height: 1024" + "extras": "width: 1024, height: 1024", + "size": 17.40, + "date": "2024 July" }, "Kandinsky 2.1": { "path": "kandinsky-community/kandinsky-2-1", "desc": "Kandinsky 2.1 is a text-conditional diffusion model based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder. Kandinsky 2.1 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas. It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.", "preview": "kandinsky-community--kandinsky-2-1.jpg", - "extras": "width: 768, height: 768, sampler: Default" + "extras": "width: 768, height: 768, sampler: Default", + "size": 5.15, + "date": "2023 April" }, "Kandinsky 2.2": { "path": "kandinsky-community/kandinsky-2-2-decoder", "desc": "Kandinsky 2.2 is a text-conditional diffusion model (+0.1!) based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder. Kandinsky 2.2 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas. It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.", "preview": "kandinsky-community--kandinsky-2-2-decoder.jpg", - "extras": "width: 768, height: 768, sampler: Default" + "extras": "width: 768, height: 768, sampler: Default", + "size": 5.15, + "date": "2023 July" }, "Kandinsky 3": { "path": "kandinsky-community/kandinsky-3", "desc": "Kandinsky 3.0 is an open-source text-to-image diffusion model built upon the Kandinsky2-x model family. In comparison to its predecessors, Kandinsky 3.0 incorporates more data and specifically related to Russian culture, which allows to generate pictures related to Russin culture. Furthermore, enhancements have been made to the text understanding and visual quality of the model, achieved by increasing the size of the text encoder and Diffusion U-Net models, respectively.", "preview": "kandinsky-community--kandinsky-3.jpg", "variant": "fp16", - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 27.72, + "date": "2023 November" }, "Playground v1": { "path": "playgroundai/playground-v1", "desc": "Playground v1 is a latent diffusion model that improves the overall HDR quality to get more stunning images.", "preview": "playgroundai--playground-v1.jpg", - "extras": "width: 512, height: 512, sampler: Default" + "extras": "width: 512, height: 512, sampler: Default", + "size": 4.95, + "date": "2023 December" }, "Playground v2 Small": { "path": "playgroundai/playground-v2-256px-base", @@ -578,22 +735,28 @@ "Playground v2.5": { "path": "playgroundai/playground-v2.5-1024px-aesthetic", "desc": "Playground v2.5 is a diffusion-based text-to-image generative model, and a successor to Playground v2. Playground v2.5 is the state-of-the-art open-source model in aesthetic quality.", - "preview": "playgroundai--playground-v2_5-1024px-aesthetic.jpg", + "preview": "playgroundai--playground-v2.5-1024px-aesthetic.jpg", "variant": "fp16", - "extras": "sampler: DPM++ 2M EDM" + "extras": "sampler: DPM++ 2M EDM", + "size": 13.35, + "date": "2023 December" }, "CogView 4": { "path": "zai-org/CogView4-6B", "desc": "An innovative cascaded framework that enhances the performance of text-to-image diffusion. CogView is the first model implementing relay diffusion in the realm of text-to-image generation, executing the task by first creating low-resolution images and subsequently applying relay-based super-resolution.", "preview": "THUDM--CogView4-6B.jpg", - "skip": true + "skip": true, + "size": 30.39, + "date": "2025 March" }, "CogView 3 Plus": { "path": "zai-org/CogView3-Plus-3B", "desc": "An innovative cascaded framework that enhances the performance of text-to-image diffusion. CogView is the first model implementing relay diffusion in the realm of text-to-image generation, executing the task by first creating low-resolution images and subsequently applying relay-based super-resolution.", "preview": "THUDM--CogView3-Plus-3B.jpg", - "skip": true + "skip": true, + "size": 24.96, + "date": "2024 October" }, "ShuttleAI Shuttle 3.0 Diffusion": { @@ -619,14 +782,18 @@ "path": "briaai/BRIA-3.2", "desc": "Bria 3.2 is the next-generation commercial-ready text-to-image model. With just 4 billion parameters, it provides exceptional aesthetics and text rendering, evaluated to provide on par results to leading open-source models, and outperforming other licensed models.", "preview": "briaai--BRIA-3.2.jpg", - "skip": true + "skip": true, + "size": 18.66, + "date": "2025 June" }, "Meissonic": { "path": "MeissonFlow/Meissonic", "desc": "Meissonic is a non-autoregressive mask image modeling text-to-image synthesis model that can generate high-resolution images. It is designed to run on consumer graphics cards.", "preview": "MeissonFlow--Meissonic.jpg", - "skip": true + "skip": true, + "size": 3.64, + "date": "2024 October" }, "aMUSEd 256": { @@ -647,7 +814,9 @@ "path": "warp-ai/wuerstchen", "desc": "Würstchen is a diffusion model whose text-conditional model works in a highly compressed latent space of images. Why is this important? Compressing data can reduce computational costs for both training and inference by magnitudes. Training on 1024x1024 images, is way more expensive than training at 32x32. Usually, other works make use of a relatively small compression, in the range of 4x - 8x spatial compression. Würstchen takes this to an extreme. Through its novel design, we achieve a 42x spatial compression. Würstchen employs a two-stage compression, what we call Stage A and Stage B. Stage A is a VQGAN, and Stage B is a Diffusion Autoencoder (more details can be found in the paper). A third model, Stage C, is learned in that highly compressed latent space. This training requires fractions of the compute used for current top-performing models, allowing also cheaper and faster inference.", "preview": "warp-ai--wuerstchen.jpg", - "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 0.0" + "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 0.0", + "size": 12.16, + "date": "2023 August" }, "KOALA 700M": { @@ -656,7 +825,9 @@ "skip": true, "desc": "Fast text-to-image model, called KOALA, by compressing SDXL's U-Net and distilling knowledge from SDXL into our model. KOALA-700M can generate a 1024x1024 image in less than 1.5 seconds on an NVIDIA 4090 GPU, which is more than 2x faster than SDXL.", "preview": "etri-vilab--koala-700m-llava-cap.jpg", - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 6.58, + "date": "2024 January" }, "HDM-XUT 340M Anime": { @@ -671,13 +842,17 @@ "path": "thu-ml/unidiffuser-v1", "desc": "UniDiffuser is a unified diffusion framework to fit all distributions relevant to a set of multi-modal data in one transformer. UniDiffuser is able to perform image, text, text-to-image, image-to-text, and image-text pair generation by setting proper timesteps without additional overhead.\nSpecifically, UniDiffuser employs a variation of transformer, called U-ViT, which parameterizes the joint noise prediction network. Other components perform as encoders and decoders of different modalities, including a pretrained image autoencoder from Stable Diffusion, a pretrained image ViT-B/32 CLIP encoder, a pretrained text ViT-L CLIP encoder, and a GPT-2 text decoder finetuned by ourselves.", "preview": "thu-ml--unidiffuser-v1.jpg", - "extras": "width: 512, height: 512, sampler: Default" + "extras": "width: 512, height: 512, sampler: Default", + "size": 5.37, + "date": "2023 May" }, "SalesForce BLIP-Diffusion": { "path": "salesforce/blipdiffusion", "desc": "BLIP-Diffusion, a new subject-driven image generation model that supports multimodal control which consumes inputs of subject images and text prompts. Unlike other subject-driven generation models, BLIP-Diffusion introduces a new multimodal encoder which is pre-trained to provide subject representation.", - "preview": "salesforce--blipdiffusion.jpg" + "preview": "salesforce--blipdiffusion.jpg", + "size": 7.23, + "date": "2023 July" }, "InstaFlow 0.9B": { @@ -690,13 +865,17 @@ "path": "DeepFloyd/IF-I-M-v1.0", "desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.", "preview": "DeepFloyd--IF-I-M-v1.0.jpg", - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 12.79, + "date": "2023 April" }, "DeepFloyd IF Large": { "path": "DeepFloyd/IF-I-L-v1.0", "desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.", "preview": "DeepFloyd--IF-I-L-v1.0.jpg", - "extras": "sampler: Default" + "extras": "sampler: Default", + "size": 15.48, + "date": "2023 April" } } diff --git a/installer.py b/installer.py index 8609ff953..cde81c9c2 100644 --- a/installer.py +++ b/installer.py @@ -56,6 +56,7 @@ args = Dot({ }) git_commit = "unknown" diffusers_commit = "unknown" +restart_required = False extensions_commit = { # force specific commit for extensions 'sd-webui-controlnet': 'ecd33eb', 'adetailer': 'a89c01d' @@ -345,6 +346,8 @@ def installed(package, friendly: str = None, reload = False, quiet = False): # p log.warning(f'Install: package="{p[0]}" installed={pkg_version} required={p[1]} allowing experimental') else: log.warning(f'Install: package="{p[0]}" installed={pkg_version} required={p[1]} version mismatch') + global restart_required # pylint: disable=global-statement + restart_required = True ok = ok and (exact or args.experimental) else: if not quiet: @@ -605,7 +608,9 @@ def check_diffusers(): if args.skip_git: install('diffusers') return - sha = '5e181eddfe7e44c1444a2511b0d8e21d177850a0' # diffusers commit hash + sha = '64a5187d96f9376c7cf5123db810f2d2da79d7d0' # diffusers commit hash + if args.use_rocm or args.use_zluda or args.use_directml: + sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else -1) cur = opts.get('diffusers_version', '') if minor > -1 else '' @@ -626,19 +631,19 @@ def check_transformers(): t_start = time.time() if args.skip_all or args.skip_git or args.experimental: return - pkg_transofmers = pkg_resources.working_set.by_key.get('transformers', None) + pkg_transformers = pkg_resources.working_set.by_key.get('transformers', None) pkg_tokenizers = pkg_resources.working_set.by_key.get('tokenizers', None) if args.use_directml: target_transformers = '4.52.4' target_tokenizers = '0.21.4' else: - target_transformers = '4.56.1' - target_tokenizers = '0.22.0' - if (pkg_transofmers is None) or ((pkg_transofmers.version != target_transformers) or (pkg_tokenizers is None) or ((pkg_tokenizers.version != target_tokenizers) and (not args.experimental))): - if pkg_transofmers is None: + target_transformers = '4.56.2' + target_tokenizers = '0.22.1' + if (pkg_transformers is None) or ((pkg_transformers.version != target_transformers) or (pkg_tokenizers is None) or ((pkg_tokenizers.version != target_tokenizers) and (not args.experimental))): + if pkg_transformers is None: log.info(f'Transformers install: version={target_transformers}') else: - log.info(f'Transformers update: current={pkg_transofmers.version} target={target_transformers}') + log.info(f'Transformers update: current={pkg_transformers.version} target={target_transformers}') pip('uninstall --yes transformers', ignore=True, quiet=True, uv=False) pip(f'install --upgrade tokenizers=={target_tokenizers}', ignore=False, quiet=True, uv=False) pip(f'install --upgrade transformers=={target_transformers}', ignore=False, quiet=True, uv=False) @@ -674,82 +679,83 @@ def install_rocm_zluda(): if args.skip_all or args.skip_requirements: return torch_command from modules import rocm - if not rocm.is_installed: - log.warning('ROCm: could not find ROCm toolkit installed') - log.info('Using CPU-only torch') - return os.environ.get('TORCH_COMMAND', 'torch torchvision') - log.info('ROCm: AMD toolkit detected') - # if not is_windows: - # os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow-rocm') - - device = None + amd_gpus = [] try: - amd_gpus = rocm.get_agents() - if len(amd_gpus) == 0: - log.warning('ROCm: no agent was found') + if sys.platform == "win32" and not rocm.is_installed: + amd_gpus = rocm.driver_get_agents() else: - log.info(f'ROCm: agents={[gpu.name for gpu in amd_gpus]}') - if args.device_id is None: - index = 0 - for idx, gpu in enumerate(amd_gpus): - index = idx - # if gpu.name.startswith('gfx11') and os.environ.get('TENSORFLOW_PACKAGE') == 'tensorflow-rocm': # do not use tensorflow-rocm for navi 3x - # os.environ['TENSORFLOW_PACKAGE'] = 'tensorflow==2.13.0' - if not gpu.is_apu: - # although apu was found, there can be a dedicated card. do not break loop. - # if no dedicated card was found, apu will be used. - break - os.environ.setdefault('HIP_VISIBLE_DEVICES', str(index)) - device = amd_gpus[index] - else: - device_id = int(args.device_id) - if device_id < len(amd_gpus): - device = amd_gpus[device_id] + amd_gpus = rocm.get_agents() + log.info('ROCm: AMD toolkit detected') except Exception as e: log.warning(f'ROCm agent enumerator failed: {e}') + #os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow') + + device = None + if len(amd_gpus) == 0: + log.warning('ROCm: no agent was found') + else: + log.info(f'ROCm: agents={[gpu.name for gpu in amd_gpus]}') + if args.device_id is None: + index = 0 + for idx, gpu in enumerate(amd_gpus): + index = idx + if not gpu.is_apu: + # although apu was found, there can be a dedicated card. do not break loop. + # if no dedicated card was found, apu will be used. + break + os.environ.setdefault('HIP_VISIBLE_DEVICES', str(index)) + device = amd_gpus[index] + else: + device_id = int(args.device_id) + if device_id < len(amd_gpus): + device = amd_gpus[device_id] + + if sys.platform == "win32" and args.use_rocm and not rocm.is_installed: + check_python(supported_minors=[11, 12, 13], reason='ROCm backend requires a Python version between 3.11 and 3.13') + install(f"rocm rocm-sdk-core --index-url https://rocm.nightlies.amd.com/v2-staging/{device.therock}") + rocm.refresh() + msg = f'ROCm: version={rocm.version}' if device is not None: msg += f', using agent {device.name}' log.info(msg) - if sys.platform == "win32": # TODO install: enable ROCm for windows when available - #check_python(supported_minors=[10, 11, 12, 13], reason='ZLUDA backend requires a Python version between 3.10 and 3.13') + if sys.platform == "win32": + if args.use_rocm: # TODO install: switch to pytorch source when it becomes available + if device is not None and isinstance(rocm.environment, rocm.PythonPackageEnvironment): # TheRock + check_python(supported_minors=[11, 12, 13], reason='ROCm backend requires a Python version between 3.11 and 3.13') + torch_command = os.environ.get('TORCH_COMMAND', f'torch torchvision --index-url https://rocm.nightlies.amd.com/v2-staging/{device.therock}') + else: + check_python(supported_minors=[12], reason='ROCm Windows preview requires Python version 3.12') + torch_command = os.environ.get('TORCH_COMMAND', '--no-cache-dir https://repo.radeon.com/rocm/windows/rocm-rel-6.4.4/torch-2.8.0a0%2Bgitfc14c65-cp312-cp312-win_amd64.whl https://repo.radeon.com/rocm/windows/rocm-rel-6.4.4/torchvision-0.24.0a0%2Bc85f008-cp312-cp312-win_amd64.whl') + else: + #check_python(supported_minors=[10, 11, 12, 13], reason='ZLUDA backend requires a Python version between 3.10 and 3.13') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1+cu118 torchvision==0.22.1+cu118 --index-url https://download.pytorch.org/whl/cu118') - if args.device_id is not None: - if os.environ.get('HIP_VISIBLE_DEVICES', None) is not None: - log.warning('Setting HIP_VISIBLE_DEVICES and --device-id at the same time may be mistake.') - os.environ['HIP_VISIBLE_DEVICES'] = args.device_id - del args.device_id + if args.device_id is not None: + if os.environ.get('HIP_VISIBLE_DEVICES', None) is not None: + log.warning('Setting HIP_VISIBLE_DEVICES and --device-id at the same time may be mistake.') + os.environ['HIP_VISIBLE_DEVICES'] = args.device_id + del args.device_id - error = None - from modules import zluda_installer - try: - if args.reinstall or zluda_installer.is_reinstall_needed(): - zluda_installer.uninstall() - zluda_installer.install() - zluda_installer.set_default_agent(device) - except Exception as e: - error = e - log.warning(f'Failed to install ZLUDA: {e}') + from modules import zluda_installer + try: + if args.reinstall or zluda_installer.is_reinstall_needed(): + zluda_installer.uninstall() + zluda_installer.install() + zluda_installer.set_default_agent(device) + except Exception as e: + log.warning(f'Failed to install ZLUDA: {e}') - if error is None: try: zluda_installer.load() - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1+cu118 torchvision==0.22.1+cu118 --index-url https://download.pytorch.org/whl/cu118') except Exception as e: - error = e log.warning(f'Failed to load ZLUDA: {e}') - if error is not None: - log.info('Using CPU-only torch') - torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') else: #check_python(supported_minors=[10, 11, 12, 13], reason='ROCm backend requires a Python version between 3.10 and 3.13') - if os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", None) is None: - os.environ.setdefault('TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL', '1') - if args.use_nightly: if rocm.version is None or float(rocm.version) >= 6.4: # assume the latest if version check fails torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm6.4') @@ -779,7 +785,7 @@ def install_rocm_zluda(): log.info(f'ROCm: HSA_OVERRIDE_GFX_VERSION auto config skipped: device={device.name if device is not None else None} version={os.environ.get("HSA_OVERRIDE_GFX_VERSION", None)}') else: gfx_ver = device.get_gfx_version() - if gfx_ver is not None: + if gfx_ver is not None and device.name.removeprefix("gfx") != gfx_ver.replace(".", ""): os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', gfx_ver) log.info(f'ROCm: HSA_OVERRIDE_GFX_VERSION config overridden: device={device.name} version={os.environ.get("HSA_OVERRIDE_GFX_VERSION", None)}') @@ -793,27 +799,6 @@ def install_ipex(): args.use_ipex = True # pylint: disable=attribute-defined-outside-init log.info('IPEX: Intel OneAPI toolkit detected') - if os.environ.get("NEOReadDebugKeys", None) is None: - os.environ.setdefault('NEOReadDebugKeys', '1') - - if os.environ.get("ClDeviceGlobalMemSizeAvailablePercent", None) is None: - os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') - - if os.environ.get("SYCL_CACHE_PERSISTENT", None) is None: - os.environ.setdefault('SYCL_CACHE_PERSISTENT', '1') # Jit cache - - if os.environ.get("PYTORCH_ENABLE_XPU_FALLBACK", None) is None: - os.environ.setdefault('PYTORCH_ENABLE_XPU_FALLBACK', '1') # CPU fallback for unsupported ops - - if os.environ.get("UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS", None) is None: - os.environ.setdefault('UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS', '1') # Work around the 4G alloc limit on Alchemist - - # FP64 emulation causes random UR Errors - #if os.environ.get("OverrideDefaultFP64Settings", None) is None: - # os.environ.setdefault('OverrideDefaultFP64Settings', '1') - #if os.environ.get("IGC_EnableDPEmulation", None) is None: - # os.environ.setdefault('IGC_EnableDPEmulation', '1') # FP64 Emulation - if args.use_nightly: torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/xpu') else: @@ -827,20 +812,17 @@ def install_ipex(): def install_openvino(): t_start = time.time() log.info('OpenVINO: selected') - #check_python(supported_minors=[10, 11, 12, 13], reason='OpenVINO backend requires a Python version between 3.10 and 3.13') + os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX') + #check_python(supported_minors=[10, 11, 12, 13], reason='OpenVINO backend requires a Python version between 3.10 and 3.13') if sys.platform == 'darwin': torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.8.0 torchvision==0.23.0') else: torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.8.0+cpu torchvision==0.23.0 --index-url https://download.pytorch.org/whl/cpu') - install(os.environ.get('OPENVINO_COMMAND', 'openvino==2025.3.0'), 'openvino') - install(os.environ.get('NNCF_COMMAND', 'nncf==2.18.0'), 'nncf') - os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX') - if os.environ.get("NEOReadDebugKeys", None) is None: - os.environ.setdefault('NEOReadDebugKeys', '1') - if os.environ.get("ClDeviceGlobalMemSizeAvailablePercent", None) is None: - os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') + if not (args.skip_all or args.skip_requirements): + install(os.environ.get('OPENVINO_COMMAND', 'openvino==2025.3.0'), 'openvino') + install(os.environ.get('NNCF_COMMAND', 'nncf==2.18.0'), 'nncf') ts('openvino', t_start) return torch_command @@ -874,6 +856,7 @@ def install_torch_addons(): install('pillow-jxl-plugin==1.3.4', 'pillow-jxl-plugin') if not args.experimental: uninstall('wandb', quiet=True) + uninstall('pynvml', quiet=True) ts('addons', t_start) @@ -921,9 +904,9 @@ def check_torch(): if torch_command != '': pass else: - is_cuda_available = allow_cuda and (shutil.which('nvidia-smi') is not None or args.use_xformers or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))) - is_rocm_available = allow_rocm and rocm.is_installed - is_ipex_available = allow_ipex and (args.use_ipex or shutil.which('sycl-ls') is not None or shutil.which('sycl-ls.exe') is not None or os.environ.get('ONEAPI_ROOT') is not None or os.path.exists('/opt/intel/oneapi') or os.path.exists("C:/Program Files (x86)/Intel/oneAPI") or os.path.exists("C:/oneAPI")) + is_cuda_available = allow_cuda and (args.use_cuda or shutil.which('nvidia-smi') is not None or args.use_xformers or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))) + is_rocm_available = allow_rocm and (args.use_rocm or args.use_zluda or rocm.is_installed) + is_ipex_available = allow_ipex and (args.use_ipex or shutil.which('sycl-ls') is not None or shutil.which('sycl-ls.exe') is not None or os.environ.get('ONEAPI_ROOT') is not None or os.path.exists('/opt/intel/oneapi') or os.path.exists("C:/Program Files (x86)/Intel/oneAPI") or os.path.exists("C:/oneAPI") or os.path.exists("C:/Program Files/Intel/Intel Graphics Software")) if is_cuda_available and args.use_cuda: # prioritize cuda torch_command = install_cuda() @@ -950,63 +933,61 @@ def check_torch(): install(torch_command, 'torch torchvision') install('onnxruntime-directml', 'onnxruntime-directml', ignore=True) else: - if args.use_zluda: - log.warning("ZLUDA failed to initialize: no HIP SDK found") log.warning('Torch: CPU-only version installed') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') - if 'torch' in torch_command and not args.version: + if args.version: + return + + if 'torch' in torch_command: if not installed('torch'): log.info(f'Torch: download and install in progress... cmd="{torch_command}"') install('--upgrade pip', 'pip', reinstall=True) # pytorch rocm is too large for older pip install(torch_command, 'torch torchvision', quiet=True) - else: + + try: + import torch try: - import torch - log.info(f'Torch {torch.__version__}') - if args.use_ipex and allow_ipex: - try: - import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import - log.info(f'Torch backend: Intel IPEX {ipex.__version__}') - except Exception: - log.warning('IPEX: not found') - if shutil.which('icpx') is not None: - log.info(f'{os.popen("icpx --version").read().rstrip()}') - for device in range(torch.xpu.device_count()): - log.info(f'Torch detected GPU: {torch.xpu.get_device_name(device)} VRAM {round(torch.xpu.get_device_properties(device).total_memory / 1024 / 1024)} Compute Units {torch.xpu.get_device_properties(device).max_compute_units}') - elif torch.cuda.is_available() and (allow_cuda or allow_rocm): - # log.debug(f'Torch allocator: {torch.cuda.get_allocator_backend()}') - if torch.version.cuda and allow_cuda: - log.info(f'Torch backend: nVidia CUDA {torch.version.cuda} cuDNN {torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else "N/A"}') - elif torch.version.hip and allow_rocm: - log.info(f'Torch backend: AMD ROCm HIP {torch.version.hip}') - else: - log.warning('Unknown Torch backend') - for device in [torch.cuda.device(i) for i in range(torch.cuda.device_count())]: - log.info(f'Torch detected GPU: {torch.cuda.get_device_name(device)} VRAM {round(torch.cuda.get_device_properties(device).total_memory / 1024 / 1024)} Arch {torch.cuda.get_device_capability(device)} Cores {torch.cuda.get_device_properties(device).multi_processor_count}') + import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import + log.info(f'Torch backend: type=IPEX version={ipex.__version__}') + except Exception: + pass + if 'cpu' in torch.__version__: + if is_cuda_available: + log.warning(f'Torch: version="{torch.__version__}" CPU version installed and CUDA is available - consider reinstalling') + elif is_rocm_available: + log.warning(f'Torch: version="{torch.__version__}" CPU version installed and ROCm is available - consider reinstalling') + if hasattr(torch, "xpu") and torch.xpu.is_available() and allow_ipex: + if shutil.which('icpx') is not None: + log.info(f'{os.popen("icpx --version").read().rstrip()}') + for device in range(torch.xpu.device_count()): + log.info(f'Torch detected: gpu="{torch.xpu.get_device_name(device)}" vram={round(torch.xpu.get_device_properties(device).total_memory / 1024 / 1024)} units={torch.xpu.get_device_properties(device).max_compute_units}') + elif torch.cuda.is_available() and (allow_cuda or allow_rocm): + if torch.version.cuda and allow_cuda: + log.info(f'Torch backend: version="{torch.__version__}" type=CUDA CUDA={torch.version.cuda} cuDNN={torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else "N/A"}') + elif torch.version.hip and allow_rocm: + log.info(f'Torch backend: version="{torch.__version__}" type=ROCm HIP={torch.version.hip}') else: - try: - if args.use_directml and allow_directml: - import torch_directml # pylint: disable=import-error - dml_ver = pkg_resources.get_distribution("torch-directml") - log.info(f'Torch backend: DirectML ({dml_ver})') - for i in range(0, torch_directml.device_count()): - log.info(f'Torch detected GPU: {torch_directml.device_name(i)}') - except Exception: - log.warning("Torch reports CUDA not available") - except Exception as e: - log.error(f'Torch cannot load: {e}') - if not args.ignore: - sys.exit(1) - if rocm.is_installed: - if sys.platform == "win32": # CPU, DirectML, ZLUDA - rocm.conceal() - elif rocm.is_wsl: # WSL ROCm + log.warning('Unknown Torch backend') + for device in [torch.cuda.device(i) for i in range(torch.cuda.device_count())]: + log.info(f'Torch detected: gpu="{torch.cuda.get_device_name(device)}" vram={round(torch.cuda.get_device_properties(device).total_memory / 1024 / 1024)} arch={torch.cuda.get_device_capability(device)} cores={torch.cuda.get_device_properties(device).multi_processor_count}') + else: try: - rocm.load_hsa_runtime() - except OSError: - log.error("ROCm: failed to preload HSA runtime") - if args.version: - return + if args.use_directml and allow_directml: + import torch_directml # pylint: disable=import-error + dml_ver = pkg_resources.get_distribution("torch-directml") + log.warning(f'Torch backend: DirectML ({dml_ver})') + log.warning('DirectML: end-of-life') + for i in range(0, torch_directml.device_count()): + log.info(f'Torch detected GPU: {torch_directml.device_name(i)}') + except Exception: + log.warning("Torch reports CUDA not available") + except Exception as e: + log.error(f'Torch cannot load: {e}') + if not args.ignore: + sys.exit(1) + + if rocm.is_installed: + rocm.postinstall() if not args.skip_all: install_torch_addons() check_cudnn() @@ -1264,6 +1245,17 @@ def install_pydantic(): reload('pydantic', '1.10.21') +def install_insightface(): + install('git+https://github.com/deepinsight/insightface@29b6cd65aa0e9ae3b6602de3c52e9d8949c8ee86#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches + if args.new: + uninstall('albumentations') + install('albumentationsx') + else: + uninstall('albumentationsx') + install('albumentations==1.4.3', ignore=True, quiet=True) + install_pydantic() + + def install_optional(): t_start = time.time() log.info('Installing optional requirements...') @@ -1277,8 +1269,6 @@ def install_optional(): install('nvidia-ml-py', ignore=True, quiet=True) install('ultralytics==8.3.40', ignore=True, quiet=True) install('Cython', ignore=True, quiet=True) - install('git+https://github.com/deepinsight/insightface@554a05561cb71cfebb4e012dfea48807f845a0c2#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches - install('albumentations==1.4.3', ignore=True, quiet=True) install('av', ignore=True, quiet=True) install('gguf', ignore=True) try: @@ -1358,8 +1348,14 @@ def set_environment(): os.environ.setdefault('PYTORCH_CUDA_ALLOC_CONF', allocator) os.environ.setdefault('PYTORCH_HIP_ALLOC_CONF', allocator) log.debug(f'Torch allocator: "{allocator}"') - if sys.platform == 'darwin': - os.environ.setdefault('PYTORCH_ENABLE_MPS_FALLBACK', '1') + os.environ.setdefault('TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL', '1') + os.environ.setdefault('NEOReadDebugKeys', '1') + os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') + os.environ.setdefault('SYCL_CACHE_PERSISTENT', '1') + os.environ.setdefault('UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS', '1') + os.environ.setdefault('PYTORCH_ENABLE_XPU_FALLBACK', '1') + os.environ.setdefault('PYTORCH_ENABLE_MPS_FALLBACK', '1') + os.environ.setdefault('TOKENIZERS_PARALLELISM', '0') def check_extensions(): @@ -1634,7 +1630,6 @@ def add_args(parser): group_http.add_argument("--cors-regex", type=str, default=os.environ.get("SD_CORSREGEX", None), help="Allowed CORS origins as regular expression, default: %(default)s") group_http.add_argument('--subpath', type=str, default=os.environ.get("SD_SUBPATH", None), help='Customize the URL subpath for usage with reverse proxy') group_http.add_argument("--autolaunch", default=os.environ.get("SD_AUTOLAUNCH", False), action='store_true', help="Open the UI URL in the system's default browser upon launch") - group_http.add_argument('--docs', default=os.environ.get("SD_DOCS", False), action='store_true', help = "Mount API docs, default: %(default)s") group_http.add_argument("--auth", type=str, default=os.environ.get("SD_AUTH", None), help='Set access authentication like "user:pwd,user:pwd""') group_http.add_argument("--auth-file", type=str, default=os.environ.get("SD_AUTHFILE", None), help='Set access authentication using file, default: %(default)s') group_http.add_argument("--allowed-paths", nargs='+', default=[], type=str, required=False, help="add additional paths to paths allowed for web access") @@ -1655,11 +1650,11 @@ def add_args(parser): group_log = parser.add_argument_group('Logging') group_log.add_argument("--log", type=str, default=os.environ.get("SD_LOG", None), help="Set log file, default: %(default)s") - group_log.add_argument('--debug', default=os.environ.get("SD_DEBUG",False), action='store_true', help="Run with debug logging, default: %(default)s") + group_log.add_argument('--debug', default=not os.environ.get("SD_NODEBUG",False), action='store_true', help="Run with debug logging, default: %(default)s") group_log.add_argument("--trace", default=os.environ.get("SD_TRACE", False), action='store_true', help="Run with trace logging, default: %(default)s") group_log.add_argument("--profile", default=os.environ.get("SD_PROFILE", False), action='store_true', help="Run profiler, default: %(default)s") - group_log.add_argument('--docs', default=os.environ.get("SD_DOCS", False), action='store_true', help="Mount API docs, default: %(default)s") - group_log.add_argument("--api-log", default=os.environ.get("SD_APILOG", False), action='store_true', help="Log all API requests") + group_log.add_argument('--docs', default=not os.environ.get("SD_NODOCS", False), action='store_true', help = "Mount API docs, default: %(default)s") + group_log.add_argument("--api-log", default=not os.environ.get("SD_NOAPILOG", False), action='store_true', help="Log all API requests") group_nargs = parser.add_argument_group('Other') group_nargs.add_argument('args', type=str, nargs='*', help=argparse.SUPPRESS) diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 3508ac572..95b5aca10 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -293,10 +293,11 @@ function extraNetworksSearchButton(event) { } function extraNetworksFilterVersion(event) { - // log('extraNetworksFilterVersion', event); + log('extraNetworksFilterVersion', event); const version = event.target.textContent.trim(); const activeTab = getENActiveTab(); - const cardContainer = gradioApp().querySelector(`#${activeTab}_model_cards`); + const activePage = getENActivePage().toLowerCase(); + const cardContainer = gradioApp().querySelector(`#${activeTab}_${activePage}_cards`); if (!cardContainer) return; if (cardContainer.dataset.activeVersion === version) { cardContainer.dataset.activeVersion = ''; diff --git a/javascript/guidance.js b/javascript/guidance.js new file mode 100644 index 000000000..11c2430d3 --- /dev/null +++ b/javascript/guidance.js @@ -0,0 +1,20 @@ +const guiders = { + None: '', + 'LSC: LayerSkipConfig': 'https://github.com/huggingface/diffusers/blob/041501aea92919c9c7f36e189fc9cf7d865ebb96/src/diffusers/hooks/layer_skip.py#L41', + 'CFG: ClassifierFreeGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.ClassifierFreeGuidance', + 'Auto: AutoGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.AutoGuidance', + 'Zero: ClassifierFreeZeroStar': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.ClassifierFreeZeroStarGuidance', + 'PAG: PerturbedAttentionGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.PerturbedAttentionGuidance', + 'APG: AdaptiveProjectedGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.AdaptiveProjectedGuidance', + 'SLG: SkipLayerGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.SkipLayerGuidance', + 'SEG: SmoothedEnergyGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.SmoothedEnergyGuidance', + 'TCFG: TangentialClassifierFreeGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.TangentialClassifierFreeGuidance', + 'FDG: FrequencyDecoupledGuidance': 'https://huggingface.co/docs/diffusers/v0.35.1/en/api/modular_diffusers/guiders#diffusers.FrequencyDecoupledGuidance', +}; + +function getGuidanceDocs(guider) { + if (guider.label) guider = guider.label; + const url = guiders[guider]; + log('getGuidanceDocs', guider, url); + if (url) window.open(url, '_blank'); +} diff --git a/javascript/logMonitor.js b/javascript/logMonitor.js index 08f104b55..21c03e88d 100644 --- a/javascript/logMonitor.js +++ b/javascript/logMonitor.js @@ -78,6 +78,7 @@ async function logMonitor() { for (const line of lines) addLogLine(line); if (!logConnected) { logConnected = true; + monitorConnection(); xhrPost(`${window.api}/log`, { debug: 'connected' }); } } else { diff --git a/javascript/monitor.js b/javascript/monitor.js new file mode 100644 index 000000000..276e4bc0e --- /dev/null +++ b/javascript/monitor.js @@ -0,0 +1,35 @@ +async function updateIndicator(online, data, msg) { + const el = document.getElementById('logo_nav'); + if (!el || !data) return; + const status = online ? 'online' : 'offline'; + const date = new Date(); + const template = ` + Version: ${data.updated}
+ Commit: ${data.hash}
+ Branch: ${data.branch}
+ Status: ${status}
+ Since: ${date.toLocaleString()}
+ `; + if (online) { + el.dataset.hint = template; + el.style.backgroundColor = 'var(--sd-main-accent-color)'; + log('monitorConnection: online', data); + } else { + el.dataset.hint = template; + el.style.backgroundColor = 'var(--color-error)'; + log('monitorConnection: offline', msg); + } +} + +async function monitorConnection() { + try { + const res = await fetch(`${window.api}/version`); + const data = await res.json(); + const url = res.url.split('/sdapi')[0].replace('http', 'ws'); // update global url as ws need fqdn + const ws = new WebSocket(`${url}/queue/join`); + ws.onopen = () => updateIndicator(true, data, ''); + ws.onclose = () => updateIndicator(false, data, ''); + ws.onerror = (e) => updateIndicator(false, data, e.message); + ws.onmessage = (evt) => log('monitorConnection: message', evt.data); + } catch { /**/ } +} diff --git a/javascript/ui.js b/javascript/ui.js index a61297a54..8c5cd6630 100644 --- a/javascript/ui.js +++ b/javascript/ui.js @@ -601,4 +601,5 @@ async function reconnectUI() { const sd_model_observer = new MutationObserver(sd_model_callback); sd_model_observer.observe(sd_model, { attributes: true, childList: true, subtree: true }); log('reconnectUI'); + monitorConnection(); } diff --git a/launch.py b/launch.py index 8201b8275..1f55a848d 100755 --- a/launch.py +++ b/launch.py @@ -297,6 +297,9 @@ def main(): installer.log.info(f'Installer time: {init_summary()}') get_custom_args() + if installer.restart_required: + installer.log.warning('Restart is recommended due to packages updates...') + uv, instance = start_server(immediate=True, server=None) t_server = time.time() t_monitor = time.time() diff --git a/models/Reference/Qwen--Qwen-Image-Edit-2509.jpg b/models/Reference/Qwen--Qwen-Image-Edit-2509.jpg new file mode 100644 index 000000000..6209b7ba0 Binary files /dev/null and b/models/Reference/Qwen--Qwen-Image-Edit-2509.jpg differ diff --git a/models/Reference/Qwen-Image-Distill.jpg b/models/Reference/Qwen-Image-Distill.jpg deleted file mode 100644 index ac05ebcee..000000000 Binary files a/models/Reference/Qwen-Image-Distill.jpg and /dev/null differ diff --git a/models/Reference/briaai--BRIA-3.2.jpg b/models/Reference/briaai--BRIA-3.2.jpg index dc7067ec5..91476ba26 100644 Binary files a/models/Reference/briaai--BRIA-3.2.jpg and b/models/Reference/briaai--BRIA-3.2.jpg differ diff --git a/models/Reference/fal--AuraFlow-v0.3.jpg b/models/Reference/fal--AuraFlow-v0.3.jpg index 1ce8fe728..beff71e97 100644 Binary files a/models/Reference/fal--AuraFlow-v0.3.jpg and b/models/Reference/fal--AuraFlow-v0.3.jpg differ diff --git a/models/Reference/juggernautXL_v9Rundiffusionphoto2.jpg b/models/Reference/juggernautXL_juggXIByRundiffusion.jpg similarity index 100% rename from models/Reference/juggernautXL_v9Rundiffusionphoto2.jpg rename to models/Reference/juggernautXL_juggXIByRundiffusion.jpg diff --git a/models/Reference/juggernautXL_v9Rdphoto2Lightning.jpg b/models/Reference/juggernautXL_juggXILightningByRD.jpg similarity index 100% rename from models/Reference/juggernautXL_v9Rdphoto2Lightning.jpg rename to models/Reference/juggernautXL_juggXILightningByRD.jpg diff --git a/models/Reference/kandinsky-community--kandinsky-2-1.jpg b/models/Reference/kandinsky-community--kandinsky-2-1.jpg index 2fb28d1a4..bebbd6379 100644 Binary files a/models/Reference/kandinsky-community--kandinsky-2-1.jpg and b/models/Reference/kandinsky-community--kandinsky-2-1.jpg differ diff --git a/models/Reference/kandinsky-community--kandinsky-2-2-decoder.jpg b/models/Reference/kandinsky-community--kandinsky-2-2-decoder.jpg index 743fe024c..75f7ba309 100644 Binary files a/models/Reference/kandinsky-community--kandinsky-2-2-decoder.jpg and b/models/Reference/kandinsky-community--kandinsky-2-2-decoder.jpg differ diff --git a/models/Reference/kandinsky-community--kandinsky-3.jpg b/models/Reference/kandinsky-community--kandinsky-3.jpg index f54af2b8c..933463d44 100644 Binary files a/models/Reference/kandinsky-community--kandinsky-3.jpg and b/models/Reference/kandinsky-community--kandinsky-3.jpg differ diff --git a/models/Reference/linoyts--Wan2.2-VACE-Fun-14B-diffusers.jpg b/models/Reference/linoyts--Wan2.2-VACE-Fun-14B-diffusers.jpg new file mode 100644 index 000000000..dbde7389b Binary files /dev/null and b/models/Reference/linoyts--Wan2.2-VACE-Fun-14B-diffusers.jpg differ diff --git a/models/Reference/playgroundai--playground-v2_5-1024px-aesthetic.jpg b/models/Reference/playgroundai--playground-v2.5-1024px-aesthetic.jpg similarity index 100% rename from models/Reference/playgroundai--playground-v2_5-1024px-aesthetic.jpg rename to models/Reference/playgroundai--playground-v2.5-1024px-aesthetic.jpg diff --git a/models/Reference/segmind--SSD-1B.jpg b/models/Reference/segmind--SSD-1B.jpg index 51907db27..11a96eedb 100644 Binary files a/models/Reference/segmind--SSD-1B.jpg and b/models/Reference/segmind--SSD-1B.jpg differ diff --git a/models/Reference/segmind--SegMoE-4x2-v0.jpg b/models/Reference/segmind--SegMoE-4x2-v0.jpg index 93903538c..3e9ba4ea3 100644 Binary files a/models/Reference/segmind--SegMoE-4x2-v0.jpg and b/models/Reference/segmind--SegMoE-4x2-v0.jpg differ diff --git a/models/Reference/segmind--SegMoE-SD-4x2-v0.jpg b/models/Reference/segmind--SegMoE-SD-4x2-v0.jpg index 7d7989ec2..24b738c02 100644 Binary files a/models/Reference/segmind--SegMoE-SD-4x2-v0.jpg and b/models/Reference/segmind--SegMoE-SD-4x2-v0.jpg differ diff --git a/models/Reference/segmind--Segmind-Vega.jpg b/models/Reference/segmind--Segmind-Vega.jpg index 98a5ec811..5204f1712 100644 Binary files a/models/Reference/segmind--Segmind-Vega.jpg and b/models/Reference/segmind--Segmind-Vega.jpg differ diff --git a/models/Reference/segmind--tiny-sd.jpg b/models/Reference/segmind--tiny-sd.jpg index 375f697aa..953407d56 100644 Binary files a/models/Reference/segmind--tiny-sd.jpg and b/models/Reference/segmind--tiny-sd.jpg differ diff --git a/models/Reference/stabilityai--sdxl-turbo.jpg b/models/Reference/stabilityai--sdxl-turbo.jpg new file mode 100644 index 000000000..122f3678a Binary files /dev/null and b/models/Reference/stabilityai--sdxl-turbo.jpg differ diff --git a/models/Reference/sd_xl_base_1.0.jpg b/models/Reference/stabilityai--stable-diffusion-xl-base-1.0.jpg similarity index 100% rename from models/Reference/sd_xl_base_1.0.jpg rename to models/Reference/stabilityai--stable-diffusion-xl-base-1.0.jpg diff --git a/models/Reference/tempest-by-vlad-base.jpg b/models/Reference/tempest-by-vlad-base.jpg deleted file mode 100644 index 0d48f0a62..000000000 Binary files a/models/Reference/tempest-by-vlad-base.jpg and /dev/null differ diff --git a/models/Reference/tempest-by-vlad-hyper.jpg b/models/Reference/tempest-by-vlad-hyper.jpg deleted file mode 100644 index c2ffc23e8..000000000 Binary files a/models/Reference/tempest-by-vlad-hyper.jpg and /dev/null differ diff --git a/models/Reference/tempestByVlad_baseV01.jpg b/models/Reference/tempestByVlad_baseV01.jpg new file mode 100644 index 000000000..2db7f03da Binary files /dev/null and b/models/Reference/tempestByVlad_baseV01.jpg differ diff --git a/models/Reference/tempestByVlad_hyperV01.jpg b/models/Reference/tempestByVlad_hyperV01.jpg new file mode 100644 index 000000000..dc28c29d8 Binary files /dev/null and b/models/Reference/tempestByVlad_hyperV01.jpg differ diff --git a/models/Reference/vladmandic--flux.1-dev-SRPO.jpg b/models/Reference/vladmandic--flux.1-dev-SRPO.jpg new file mode 100644 index 000000000..986483517 Binary files /dev/null and b/models/Reference/vladmandic--flux.1-dev-SRPO.jpg differ diff --git a/modules/cachedit.py b/modules/cachedit.py new file mode 100644 index 000000000..057b23804 --- /dev/null +++ b/modules/cachedit.py @@ -0,0 +1,64 @@ +import os +from installer import install +from modules import shared + + +def apply_cache_dit(pipe): + if not shared.opts.cache_dit_enabled: + return + install('git+https://github.com/vipshop/cache-dit', 'cache_dit') + os.environ.setdefault("CACHE_DIT_LOG_LEVEL", "error") + try: + import cache_dit + except Exception as e: + shared.log.error(f'Cache-DIT: {e}') + return + _, supported = cache_dit.supported_pipelines() + supported = [s.replace('*', '') for s in supported] + if not any(pipe.__class__.__name__.startswith(s) for s in supported): + shared.log.error(f'Cache-DiT: pipeline={pipe.__class__.__name__} unsupported') + return + + if getattr(pipe, 'has_cache_dit', False): + unapply_cache_dir(pipe) + + config_args = {} + if shared.opts.cache_dit_fcompute >= 0: + config_args['Fn_compute_blocks'] = int(shared.opts.cache_dit_fcompute) + if shared.opts.cache_dit_bcompute >= 0: + config_args['Bn_compute_blocks'] = int(shared.opts.cache_dit_bcompute) + if shared.opts.cache_dit_threshold >= 0: + config_args['residual_diff_threshold'] = float(shared.opts.cache_dit_threshold) + if shared.opts.cache_dit_warmup >= 0: + config_args['max_warmup_steps'] = int(shared.opts.cache_dit_warmup) + cache_config = cache_dit.BasicCacheConfig(**config_args) + if shared.opts.cache_dit_calibrator == "TaylorSeer": + calibrator_config = cache_dit.TaylorSeerCalibratorConfig(taylorseer_order=1) + elif shared.opts.cache_dit_calibrator == "FoCa": + calibrator_config = cache_dit.FoCaCalibratorConfig() + else: + calibrator_config = None + shared.log.info(f'Apply Cache-DiT: config="{cache_config.strify()}" calibrator="{calibrator_config.strify() if calibrator_config else "None"}"') + try: + cache_dit.enable_cache( + pipe, + cache_config=cache_config, + calibrator_config=calibrator_config, + ) + shared.sd_model.has_cache_dit = True + except Exception as e: + shared.log.error(f'Cache-DiT: {e}') + return + + +def unapply_cache_dir(pipe): + if not shared.opts.cache_dit_enabled or not getattr(pipe, 'has_cache_dit', False): + return + try: + import cache_dit + # stats = cache_dit.summary(pipe) + # shared.log.critical(f'Unapply Cache-DiT: {stats}') + cache_dit.disable_cache(pipe) + pipe.has_cache_dit = False + except Exception: + return diff --git a/modules/civitai/api_civitai.py b/modules/civitai/api_civitai.py index 92add902f..4041f3f36 100644 --- a/modules/civitai/api_civitai.py +++ b/modules/civitai/api_civitai.py @@ -54,6 +54,15 @@ def get_civitai( return JSONResponse(content=[], status_code=200) +def post_civitai(page:str=None): + from modules.civitai import metadata_civitai + result = [] + for r in metadata_civitai.civit_search_metadata(title=page, raw=True): + result = r # get the last yielded result + return result + + def register_api(): from modules.shared import api api.add_api_route("/sdapi/v1/civitai", get_civitai, methods=["GET"], response_model=list) + api.add_api_route("/sdapi/v1/civitai", post_civitai, methods=["POST"], response_model=list) diff --git a/modules/civitai/download_civitai.py b/modules/civitai/download_civitai.py index 508bd0a8f..d12a00830 100644 --- a/modules/civitai/download_civitai.py +++ b/modules/civitai/download_civitai.py @@ -103,10 +103,11 @@ def download_civit_model_thread(model_name: str, model_url: str, model_path: str if os.path.isfile(temp_file): starting_pos = os.path.getsize(temp_file) headers['Range'] = f'bytes={starting_pos}-' - if token is None or len(token) == 0: - token = shared.opts.civitai_token - if token is not None and len(token) > 0: - headers['Authorization'] = f'Bearer {token}' + if 'civit' in model_url.lower(): # downloader can be used for other urls too + if token is None or len(token) == 0: + token = shared.opts.civitai_token + if (token is not None) and (len(token) > 0): + headers['Authorization'] = f'Bearer {token}' r = shared.req(model_url, headers=headers, stream=True) total_size = int(r.headers.get('content-length', 0)) diff --git a/modules/civitai/metadata_civitai.py b/modules/civitai/metadata_civitai.py index 6d4315860..fab22e78f 100644 --- a/modules/civitai/metadata_civitai.py +++ b/modules/civitai/metadata_civitai.py @@ -26,7 +26,7 @@ class CivitModel: self.status = 'Not found' -def civit_update_metadata(): +def civit_update_metadata(raw:bool=False): def create_update_metadata_table(rows: list[CivitModel]): html = """ @@ -102,8 +102,8 @@ def civit_update_metadata(): model.status = 'Update available' break results.append(model) - yield create_update_metadata_table(results) - return create_update_metadata_table(results) + yield results if raw else create_update_metadata_table(results) + yield results if raw else create_update_metadata_table(results) def civit_search_model(name, tag, model_type): @@ -224,7 +224,7 @@ def atomic_civit_search_metadata(item, results): results.append(result) -def civit_search_metadata(title: str = None): +def civit_search_metadata(title: str = None, raw: bool = False): def create_search_metadata_table(rows): html = """
@@ -258,7 +258,7 @@ def civit_search_metadata(title: str = None): re_skip = [r.strip() for r in opts.extra_networks_scan_skip.split(',') if len(r.strip()) > 0] for page in get_pages(): if type(title) == str: - if page.title != title: + if page.title.lower() != title.lower(): continue if page.name == 'style' or page.name == 'wildcards': continue @@ -278,8 +278,8 @@ def civit_search_metadata(title: str = None): future_items[executor.submit(atomic_civit_search_metadata, fn, results)] = fn for future in concurrent.futures.as_completed(future_items): future.result() - yield create_search_metadata_table(results) + yield results if raw else create_search_metadata_table(results) t1 = time.time() log.debug(f'CivitAI search metadata: scanned={scanned} skipped={skipped} time={t1-t0:.2f}') - yield create_search_metadata_table(results) + yield results if raw else create_search_metadata_table(results) diff --git a/modules/control/run.py b/modules/control/run.py index 6d1b885b7..3f6b0f7be 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -255,6 +255,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg prompt: str = '', negative_prompt: str = '', styles: List[str] = [], steps: int = 20, sampler_index: int = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, + guidance_name: str = 'Default', guidance_scale: float = 6.0, guidance_rescale: float = 0.0, guidance_start: float = 0.0, guidance_stop: float = 1.0, cfg_scale: float = 6.0, clip_skip: float = 1.0, image_cfg_scale: float = 6.0, diffusers_guidance_rescale: float = 0.7, pag_scale: float = 0.0, pag_adaptive: float = 0.5, cfg_end: float = 1.0, vae_type: str = 'Full', tiling: bool = False, hidiffusion: bool = False, detailer_enabled: bool = True, detailer_prompt: str = '', detailer_negative: str = '', detailer_steps: int = 10, detailer_strength: float = 0.3, detailer_resolution: int = 1024, @@ -306,7 +307,13 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg seed_resize_from_h = seed_resize_from_h, seed_resize_from_w = seed_resize_from_w, denoising_strength = denoising_strength, - # advanced + # modular guidance + guidance_name = guidance_name, + guidance_scale = guidance_scale, + guidance_rescale = guidance_rescale, + guidance_start = guidance_start, + guidance_stop = guidance_stop, + # legacy guidance cfg_scale = cfg_scale, cfg_end = cfg_end, clip_skip = clip_skip, @@ -314,6 +321,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg diffusers_guidance_rescale = diffusers_guidance_rescale, pag_scale = pag_scale, pag_adaptive = pag_adaptive, + # advanced vae_type = vae_type, tiling = tiling, hidiffusion = hidiffusion, @@ -396,9 +404,9 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg # hires/refine defined outside of main init vae_scale_factor = sd_vae.get_vae_scale_factor() if p.enable_hr and (p.hr_resize_x == 0 or p.hr_resize_y == 0): - p.hr_upscale_to_x, p.hr_upscale_to_y = vae_scale_factor * int(p.width_before * p.hr_scale / vae_scale_factor), vae_scale_factor * int(p.height_before * p.hr_scale / vae_scale_factor) + p.hr_upscale_to_x, p.hr_upscale_to_y = int(vae_scale_factor * int(p.width_before * p.hr_scale / vae_scale_factor)), int(vae_scale_factor * int(p.height_before * p.hr_scale / vae_scale_factor)) elif p.enable_hr and (p.hr_upscale_to_x == 0 or p.hr_upscale_to_y == 0): - p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.hr_resize_x / vae_scale_factor), vae_scale_factor * int(p.hr_resize_y / vae_scale_factor) + p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.hr_resize_x / vae_scale_factor), int(vae_scale_factor * int(p.hr_resize_y / vae_scale_factor)) global p_extra_args # pylint: disable=global-statement for k, v in p_extra_args.items(): diff --git a/modules/devices.py b/modules/devices.py index abe979c17..a98b53d02 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -49,6 +49,10 @@ def has_xpu() -> bool: return bool(hasattr(torch, 'xpu') and torch.xpu.is_available()) +def has_rocm() -> bool: + return bool(torch.version.hip is not None and torch.cuda.is_available()) + + def has_zluda() -> bool: if not cuda_ok: return False diff --git a/modules/face/insightface.py b/modules/face/insightface.py index b69648904..99be2d32e 100644 --- a/modules/face/insightface.py +++ b/modules/face/insightface.py @@ -10,11 +10,9 @@ instightface_mp = None def get_app(mp_name, threshold=0.5, resolution=640): global insightface_app, instightface_mp # pylint: disable=global-statement - from installer import install, installed, install_pydantic + from installer import install, installed, install_insightface if not installed('insightface', reload=False, quiet=True): - install('git+https://github.com/deepinsight/insightface@554a05561cb71cfebb4e012dfea48807f845a0c2#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches - install('albumentations==1.4.3', ignore=False, reinstall=True) - install_pydantic() + install_insightface() if not installed('ip_adapter', reload=False, quiet=True): install('git+https://github.com/tencent-ailab/IP-Adapter.git', 'ip_adapter', ignore=False) diff --git a/modules/framepack/framepack_load.py b/modules/framepack/framepack_load.py index 0e80e8485..ed9fdde43 100644 --- a/modules/framepack/framepack_load.py +++ b/modules/framepack/framepack_load.py @@ -124,6 +124,10 @@ def load_model(variant:str=None, pipeline:str=None, text_encoder:str=None, text_ sd_models.unload_model_weights() t0 = time.time() + sd_models.hf_auth_check(model["transformer"]["repo"]) + sd_models.hf_auth_check(model["text_encoder"]["repo"]) + sd_models.hf_auth_check(model["text_encoder_2"]["repo"]) + shared.log.debug(f'FramePack load: module=llm {model["text_encoder"]}') load_args, quant_args = model_quant.get_dit_args({}, module='TE', device_map=True) text_encoder = LlamaModel.from_pretrained(model["text_encoder"]["repo"], subfolder=model["text_encoder"]["subfolder"], cache_dir=shared.opts.hfcache_dir, **load_args, **quant_args) diff --git a/modules/framepack/framepack_worker.py b/modules/framepack/framepack_worker.py index 1b87334ec..eedbad13b 100644 --- a/modules/framepack/framepack_worker.py +++ b/modules/framepack/framepack_worker.py @@ -54,7 +54,7 @@ def worker( from modules.framepack.pipeline import hunyuan from modules.framepack.pipeline import utils - from modules.framepack.pipeline.k_diffusion_hunyuan import sample_hunyuan + from modules.framepack.pipeline import k_diffusion_hunyuan is_f1 = variant == 'forward-only' total_generated_frames = 0 @@ -244,7 +244,7 @@ def worker( transformer.initialize_teacache(enable_teacache=use_teacache, num_steps=steps, rel_l1_thresh=shared.opts.teacache_thresh) t_sample = time.time() - generated_latents = sample_hunyuan( + generated_latents = k_diffusion_hunyuan.sample_hunyuan( transformer=transformer, sampler='unipc', width=width, diff --git a/modules/framepack/pipeline/uni_pc_fm.py b/modules/framepack/pipeline/uni_pc_fm.py index 6ce7c15ce..2066cd8e3 100644 --- a/modules/framepack/pipeline/uni_pc_fm.py +++ b/modules/framepack/pipeline/uni_pc_fm.py @@ -5,7 +5,7 @@ import torch - +import numpy as np from tqdm.auto import trange @@ -13,6 +13,36 @@ def expand_dims(v, dims): return v[(...,) + (None,) * (dims - 1)] +torch_linalg_solve = None + + +def test_solver(): + from modules import devices, shared + try: + a = torch.randn(50, 50).to(device=devices.device, dtype=torch.float32) + b = torch.randn(50, 2).to(device=devices.device, dtype=torch.float32) + _x = torch.linalg.solve(a, b) + return True + except Exception as e: + shared.log.debug(f'FramePack: solver=cpu {e}') + return False + + +def linalg_solve(A, B, device): + global torch_linalg_solve # pylint: disable=global-statement + if torch_linalg_solve is None: + torch_linalg_solve = test_solver() + if torch_linalg_solve: + X = torch.linalg.solve(A, B) + return X + else: + A_np = A.float().cpu().numpy() + B_np = B.float().cpu().numpy() + X_np = np.linalg.solve(A_np, B_np) + X = torch.from_numpy(X_np).to(device=device, dtype=A.dtype) + return X + + class FlowMatchUniPC: def __init__(self, model, extra_args, variant='bh1'): self.model = model @@ -78,7 +108,7 @@ class FlowMatchUniPC: if order == 2: rhos_p = torch.tensor([0.5], device=b.device) else: - rhos_p = torch.linalg.solve(R[:-1, :-1], b[:-1]) + rhos_p = linalg_solve(R[:-1, :-1], b[:-1], x.device) else: D1s = None rhos_p = None @@ -86,7 +116,7 @@ class FlowMatchUniPC: if order == 1: rhos_c = torch.tensor([0.5], device=b.device) else: - rhos_c = torch.linalg.solve(R, b) + rhos_c = linalg_solve(R, b, x.device) x_t_ = expand_dims(t / t_prev_0, dims) * x - expand_dims(h_phi_1, dims) * model_prev_0 diff --git a/modules/img2img.py b/modules/img2img.py index 19da4b92f..4809d9fa0 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -161,10 +161,8 @@ def img2img(id_task: str, state: str, mode: int, vae_type, tiling, hidiffusion, detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, n_iter, batch_size, - cfg_scale, image_cfg_scale, - diffusers_guidance_rescale, - pag_scale, pag_adaptive, - cfg_end, + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, + cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, refiner_start, clip_skip, denoising_strength, @@ -255,6 +253,11 @@ def img2img(id_task: str, state: str, mode: int, batch_size=batch_size, n_iter=n_iter, steps=steps, + guidance_name=guidance_name, + guidance_scale=guidance_scale, + guidance_rescale=guidance_rescale, + guidance_start=guidance_start, + guidance_stop=guidance_stop, cfg_scale=cfg_scale, cfg_end=cfg_end, clip_skip=clip_skip, diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index 93f94d280..0a4e19389 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -152,7 +152,7 @@ def ipex_init(): # pylint: disable=too-many-statements # torch._int_mm directly uses onednn quantized matmul # onednn qlinear is a wrapper around onednn quantized matmul if hasattr(torch.ops, "onednn") and hasattr(torch.ops.onednn, "qlinear_pointwise"): - def onednn_mm(x: torch.Tensor, y: torch.Tensor): + def onednn_mm(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: # supports int8, fp32, fp16, and bf16 matmul with accumulation using a different dtype # int8 matmul with onednn is slower than 16 bit with dim_size < 4096 return torch.ops.onednn.qlinear_pointwise.default(x, 1.0, 0, y, torch.ones(1, device=y.device), torch.zeros(1, device=y.device), None, 1.0, 0, torch.float32, "none", [], "none") @@ -165,7 +165,7 @@ def ipex_init(): # pylint: disable=too-many-statements pass # Memory: - if 'linux' in sys.platform and "WSL2" in os.popen("uname -a").read(): + if "linux" in sys.platform and "WSL2" in os.popen("uname -a").read(): torch.xpu.empty_cache = lambda: None torch.cuda.empty_cache = torch.xpu.empty_cache diff --git a/modules/intel/ipex/attention.py b/modules/intel/ipex/attention.py index c3e8ef8ed..aacccca75 100644 --- a/modules/intel/ipex/attention.py +++ b/modules/intel/ipex/attention.py @@ -8,8 +8,8 @@ from functools import cache, wraps # ARC GPUs can't allocate more than 4GB to a single block so we slice the attention layers -dynamic_attention_slice_rate = float(os.environ.get('IPEX_SDPA_SLICE_TRIGGER_RATE', 1)) -dynamic_attention_trigger_rate = float(os.environ.get('IPEX_ATTENTION_SLICE_RATE', 0.5)) +dynamic_attention_slice_rate = float(os.environ.get("IPEX_SDPA_SLICE_TRIGGER_RATE", "1")) +dynamic_attention_trigger_rate = float(os.environ.get("IPEX_ATTENTION_SLICE_RATE", "0.5")) # Find something divisible with the input_tokens @cache diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index 9f2c5a542..8e8961476 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -1,11 +1,9 @@ -from typing import Optional - import os from functools import wraps from contextlib import nullcontext import torch import numpy as np -from modules import devices, errors +from modules import devices torch_version = torch.__version__[:4] @@ -16,38 +14,25 @@ torch_version[0], torch_version[1] = int(torch_version[0]), int(torch_version[1] device_supports_fp64 = torch.xpu.has_fp64_dtype() if hasattr(torch.xpu, "has_fp64_dtype") else torch.xpu.get_device_properties(devices.device).has_fp64 -if os.environ.get('IPEX_FORCE_ATTENTION_SLICE', '0') == '0': - if torch_version[0] > 2 or (torch_version[0] == 2 and torch_version[1] >= 7): - use_dynamic_attention = False # torch 2.7 has flash atten support - else: - use_dynamic_attention = True -else: - use_dynamic_attention = bool(os.environ.get('IPEX_FORCE_ATTENTION_SLICE', '0') == '1') - # pylint: disable=protected-access, missing-function-docstring, line-too-long, unnecessary-lambda, no-else-return -class DummyDataParallel(torch.nn.Module): # pylint: disable=missing-class-docstring, unused-argument, too-few-public-methods - def __new__(cls, module, device_ids=None, output_device=None, dim=0): # pylint: disable=unused-argument - if isinstance(device_ids, list) and len(device_ids) > 1: - errors.log.error("IPEX backend doesn't support DataParallel on multiple XPU devices") - return module.to(devices.device) - -def return_null_context(*args, **kwargs): # pylint: disable=unused-argument - return nullcontext() @property def is_cuda(self): return self.device.type == "xpu" or self.device.type == "cuda" + def check_device_type(device, device_type: str) -> bool: if device is None or type(device) not in {str, int, torch.device}: return False else: return bool(torch.device(device).type == device_type) + def check_cuda(device) -> bool: return bool(isinstance(device, int) or check_device_type(device, "cuda")) + def return_xpu(device): # keep the device instance type, aka return string if the input is string return devices.device if device is None else f"xpu:{device.split(':')[-1]}" if isinstance(device, str) and ":" in device else f"xpu:{device}" if isinstance(device, int) else torch.device(f"xpu:{device.index}" if device.index is not None else "xpu") if isinstance(device, torch.device) else "xpu" @@ -95,8 +80,8 @@ def torch_get_autocast_dtype(device_type=None): # IPEX 2.5 and above has partial support but doesn't really work most of the time. original_interpolate = torch.nn.functional.interpolate @wraps(torch.nn.functional.interpolate) -def interpolate(tensor, size=None, scale_factor=None, mode='nearest', align_corners=None, recompute_scale_factor=None, antialias=False): # pylint: disable=too-many-arguments - if mode in {'bicubic', 'bilinear'}: +def interpolate(tensor, size=None, scale_factor=None, mode="nearest", align_corners=None, recompute_scale_factor=None, antialias=False): # pylint: disable=too-many-arguments + if mode in {"bicubic", "bilinear"}: return_device = tensor.device return_dtype = tensor.dtype return original_interpolate(tensor.to("cpu", dtype=torch.float32), size=size, scale_factor=scale_factor, mode=mode, @@ -106,6 +91,32 @@ def interpolate(tensor, size=None, scale_factor=None, mode='nearest', align_corn align_corners=align_corners, recompute_scale_factor=recompute_scale_factor, antialias=antialias) +# SwinIR BF16: +original_functional_pad = torch.nn.functional.pad +@wraps(torch.nn.functional.pad) +def functional_pad(input, pad, mode="constant", value=None): + if mode == "reflect" and input.dtype == torch.bfloat16: + return original_functional_pad(input.to(torch.float32), pad, mode=mode, value=value).to(dtype=torch.bfloat16) + else: + return original_functional_pad(input, pad, mode=mode, value=value) + + +# Diffusers FreeU +original_fft_fftn = torch.fft.fftn +@wraps(torch.fft.fftn) +def fft_fftn(input, s=None, dim=None, norm=None, *, out=None): + return_dtype = input.dtype + return original_fft_fftn(input.to(dtype=torch.float32), s=s, dim=dim, norm=norm, out=out).to(dtype=return_dtype) + + +# Diffusers FreeU +original_fft_ifftn = torch.fft.ifftn +@wraps(torch.fft.ifftn) +def fft_ifftn(input, s=None, dim=None, norm=None, *, out=None): + return_dtype = input.dtype + return original_fft_ifftn(input.to(dtype=torch.float32), s=s, dim=dim, norm=norm, out=out).to(dtype=return_dtype) + + # Diffusers Float64 (Alchemist GPUs doesn't support 64 bit): original_from_numpy = torch.from_numpy @wraps(torch.from_numpy) @@ -115,6 +126,7 @@ def from_numpy(ndarray): else: return original_from_numpy(ndarray) + original_as_tensor = torch.as_tensor @wraps(torch.as_tensor) def as_tensor(data, dtype=None, device=None): @@ -126,120 +138,6 @@ def as_tensor(data, dtype=None, device=None): return original_as_tensor(data, dtype=dtype, device=device) -if not use_dynamic_attention: - original_scaled_dot_product_attention = torch.nn.functional.scaled_dot_product_attention -else: - # 32 bit attention workarounds for Alchemist: - try: - from .attention import dynamic_scaled_dot_product_attention as original_scaled_dot_product_attention - except ImportError: - original_scaled_dot_product_attention = torch.nn.functional.scaled_dot_product_attention - -@wraps(torch.nn.functional.scaled_dot_product_attention) -def scaled_dot_product_attention(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor: - if query.dtype != key.dtype: - key = key.to(dtype=query.dtype) - if query.dtype != value.dtype: - value = value.to(dtype=query.dtype) - if attn_mask is not None and query.dtype != attn_mask.dtype: - attn_mask = attn_mask.to(dtype=query.dtype) - if enable_gqa: - kwargs["enable_gqa"] = enable_gqa - result = original_scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs) - if result.dtype != query.dtype: - result = result.to(dtype=query.dtype) - return result - -# Data Type Errors: -original_torch_bmm = torch.bmm -@wraps(torch.bmm) -def torch_bmm(input, mat2, *, out=None): - if input.dtype != mat2.dtype: - mat2 = mat2.to(dtype=input.dtype) - return original_torch_bmm(input, mat2, out=out) - -# Diffusers FreeU -original_fft_fftn = torch.fft.fftn -@wraps(torch.fft.fftn) -def fft_fftn(input, s=None, dim=None, norm=None, *, out=None): - return_dtype = input.dtype - return original_fft_fftn(input.to(dtype=torch.float32), s=s, dim=dim, norm=norm, out=out).to(dtype=return_dtype) - -# Diffusers FreeU -original_fft_ifftn = torch.fft.ifftn -@wraps(torch.fft.ifftn) -def fft_ifftn(input, s=None, dim=None, norm=None, *, out=None): - return_dtype = input.dtype - return original_fft_ifftn(input.to(dtype=torch.float32), s=s, dim=dim, norm=norm, out=out).to(dtype=return_dtype) - -# A1111 FP16 -original_functional_group_norm = torch.nn.functional.group_norm -@wraps(torch.nn.functional.group_norm) -def functional_group_norm(input, num_groups, weight=None, bias=None, eps=1e-05): - if weight is not None and input.dtype != weight.data.dtype: - input = input.to(dtype=weight.data.dtype) - if bias is not None and weight is not None and bias.data.dtype != weight.data.dtype: - bias.data = bias.data.to(dtype=weight.data.dtype) - return original_functional_group_norm(input, num_groups, weight=weight, bias=bias, eps=eps) - -# A1111 BF16 -original_functional_layer_norm = torch.nn.functional.layer_norm -@wraps(torch.nn.functional.layer_norm) -def functional_layer_norm(input, normalized_shape, weight=None, bias=None, eps=1e-05): - if weight is not None and input.dtype != weight.data.dtype: - input = input.to(dtype=weight.data.dtype) - if bias is not None and weight is not None and bias.data.dtype != weight.data.dtype: - bias.data = bias.data.to(dtype=weight.data.dtype) - return original_functional_layer_norm(input, normalized_shape, weight=weight, bias=bias, eps=eps) - -# Training -original_functional_linear = torch.nn.functional.linear -@wraps(torch.nn.functional.linear) -def functional_linear(input, weight, bias=None): - if input.dtype != weight.data.dtype: - input = input.to(dtype=weight.data.dtype) - if bias is not None and bias.data.dtype != weight.data.dtype: - bias.data = bias.data.to(dtype=weight.data.dtype) - return original_functional_linear(input, weight, bias=bias) - -original_functional_conv1d = torch.nn.functional.conv1d -@wraps(torch.nn.functional.conv1d) -def functional_conv1d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): - if input.dtype != weight.data.dtype: - input = input.to(dtype=weight.data.dtype) - if bias is not None and bias.data.dtype != weight.data.dtype: - bias.data = bias.data.to(dtype=weight.data.dtype) - return original_functional_conv1d(input, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups) - -original_functional_conv2d = torch.nn.functional.conv2d -@wraps(torch.nn.functional.conv2d) -def functional_conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): - if input.dtype != weight.data.dtype: - input = input.to(dtype=weight.data.dtype) - if bias is not None and bias.data.dtype != weight.data.dtype: - bias.data = bias.data.to(dtype=weight.data.dtype) - return original_functional_conv2d(input, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups) - -# LTX Video -original_functional_conv3d = torch.nn.functional.conv3d -@wraps(torch.nn.functional.conv3d) -def functional_conv3d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): - if input.dtype != weight.data.dtype: - input = input.to(dtype=weight.data.dtype) - if bias is not None and bias.data.dtype != weight.data.dtype: - bias.data = bias.data.to(dtype=weight.data.dtype) - return original_functional_conv3d(input, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups) - -# SwinIR BF16: -original_functional_pad = torch.nn.functional.pad -@wraps(torch.nn.functional.pad) -def functional_pad(input, pad, mode='constant', value=None): - if mode == 'reflect' and input.dtype == torch.bfloat16: - return original_functional_pad(input.to(torch.float32), pad, mode=mode, value=value).to(dtype=torch.bfloat16) - else: - return original_functional_pad(input, pad, mode=mode, value=value) - - original_torch_tensor = torch.tensor @wraps(torch.tensor) def torch_tensor(data, *args, dtype=None, device=None, **kwargs): @@ -254,20 +152,20 @@ def torch_tensor(data, *args, dtype=None, device=None, **kwargs): dtype = torch.float32 return original_torch_tensor(data, *args, dtype=dtype, device=device, **kwargs) + torch.Tensor.original_Tensor_to = torch.Tensor.to @wraps(torch.Tensor.to) def Tensor_to(self, device=None, *args, **kwargs): + global device_supports_fp64 if check_cuda(device): - if not device_supports_fp64 and kwargs.get("dtype", None) == torch.float64: + device = return_xpu(device) + if not device_supports_fp64: + if kwargs.get("dtype", None) == torch.float64 and ((device is None and self.device.type == "xpu") or (device is not None and torch.device(device).type == "xpu")): kwargs["dtype"] = torch.float32 - return self.original_Tensor_to(return_xpu(device), *args, **kwargs) - else: - if not device_supports_fp64: - if kwargs.get("dtype", None) == torch.float64 and ((device is None and self.device.type == "xpu") or (device is not None and torch.device(device).type == "xpu")): - kwargs["dtype"] = torch.float32 - elif device == torch.float64 and self.device.type == "xpu": - device = torch.float32 - return self.original_Tensor_to(device, *args, **kwargs) + elif device == torch.float64 and self.device.type == "xpu": + device = torch.float32 + return self.original_Tensor_to(device, *args, **kwargs) + original_Tensor_cuda = torch.Tensor.cuda @wraps(torch.Tensor.cuda) @@ -277,6 +175,7 @@ def Tensor_cuda(self, device=None, *args, **kwargs): else: return original_Tensor_cuda(self, device, *args, **kwargs) + original_Tensor_pin_memory = torch.Tensor.pin_memory @wraps(torch.Tensor.pin_memory) def Tensor_pin_memory(self, device=None, *args, **kwargs): @@ -285,6 +184,7 @@ def Tensor_pin_memory(self, device=None, *args, **kwargs): else: return original_Tensor_pin_memory(self, device, *args, **kwargs) + original_UntypedStorage_init = torch.UntypedStorage.__init__ @wraps(torch.UntypedStorage.__init__) def UntypedStorage_init(*args, device=None, **kwargs): @@ -293,6 +193,7 @@ def UntypedStorage_init(*args, device=None, **kwargs): else: return original_UntypedStorage_init(*args, device=device, **kwargs) + if torch_version[0] > 2 or (torch_version[0] == 2 and torch_version[1] >= 4): original_UntypedStorage_to = torch.UntypedStorage.to @wraps(torch.UntypedStorage.to) @@ -310,6 +211,7 @@ if torch_version[0] > 2 or (torch_version[0] == 2 and torch_version[1] >= 4): else: return original_UntypedStorage_cuda(self, device=device, non_blocking=non_blocking, **kwargs) + original_torch_empty = torch.empty @wraps(torch.empty) def torch_empty(*args, device=None, **kwargs): @@ -318,15 +220,17 @@ def torch_empty(*args, device=None, **kwargs): else: return original_torch_empty(*args, device=device, **kwargs) + original_torch_randn = torch.randn @wraps(torch.randn) def torch_randn(*args, device=None, dtype=None, **kwargs): if dtype is bytes: dtype = None if check_cuda(device): - return original_torch_randn(*args, device=return_xpu(device), **kwargs) + return original_torch_randn(*args, device=return_xpu(device), dtype=dtype, **kwargs) else: - return original_torch_randn(*args, device=device, **kwargs) + return original_torch_randn(*args, device=device, dtype=dtype, **kwargs) + original_torch_ones = torch.ones @wraps(torch.ones) @@ -336,6 +240,7 @@ def torch_ones(*args, device=None, **kwargs): else: return original_torch_ones(*args, device=device, **kwargs) + original_torch_zeros = torch.zeros @wraps(torch.zeros) def torch_zeros(*args, device=None, **kwargs): @@ -344,6 +249,7 @@ def torch_zeros(*args, device=None, **kwargs): else: return original_torch_zeros(*args, device=device, **kwargs) + original_torch_full = torch.full @wraps(torch.full) def torch_full(*args, device=None, **kwargs): @@ -352,6 +258,7 @@ def torch_full(*args, device=None, **kwargs): else: return original_torch_full(*args, device=device, **kwargs) + original_torch_linspace = torch.linspace @wraps(torch.linspace) def torch_linspace(*args, device=None, **kwargs): @@ -360,6 +267,7 @@ def torch_linspace(*args, device=None, **kwargs): else: return original_torch_linspace(*args, device=device, **kwargs) + original_torch_eye = torch.eye @wraps(torch.eye) def torch_eye(*args, device=None, **kwargs): @@ -368,6 +276,7 @@ def torch_eye(*args, device=None, **kwargs): else: return original_torch_eye(*args, device=device, **kwargs) + original_torch_load = torch.load @wraps(torch.load) def torch_load(f, map_location=None, *args, **kwargs): @@ -376,6 +285,7 @@ def torch_load(f, map_location=None, *args, **kwargs): else: return original_torch_load(f, *args, map_location=map_location, **kwargs) + @wraps(torch.cuda.synchronize) def torch_cuda_synchronize(device=None): if check_cuda(device): @@ -383,6 +293,7 @@ def torch_cuda_synchronize(device=None): else: return torch.xpu.synchronize(device) + @wraps(torch.cuda.device) def torch_cuda_device(device): if check_cuda(device): @@ -390,6 +301,7 @@ def torch_cuda_device(device): else: return torch.xpu.device(device) + @wraps(torch.cuda.set_device) def torch_cuda_set_device(device): if check_cuda(device): @@ -397,6 +309,7 @@ def torch_cuda_set_device(device): else: torch.xpu.set_device(device) + # torch.Generator has to be a class for isinstance checks original_torch_Generator = torch.Generator class torch_Generator(original_torch_Generator): @@ -434,24 +347,14 @@ def ipex_hijacks(): torch.Generator = torch_Generator torch._C.Generator = torch_Generator - torch.backends.cuda.sdp_kernel = return_null_context - torch.nn.DataParallel = DummyDataParallel torch.UntypedStorage.is_cuda = is_cuda torch.amp.autocast_mode.autocast.__init__ = autocast_init torch.nn.functional.interpolate = interpolate - torch.nn.functional.scaled_dot_product_attention = scaled_dot_product_attention - torch.nn.functional.group_norm = functional_group_norm - torch.nn.functional.layer_norm = functional_layer_norm - torch.nn.functional.linear = functional_linear - torch.nn.functional.conv1d = functional_conv1d - torch.nn.functional.conv2d = functional_conv2d - torch.nn.functional.conv3d = functional_conv3d torch.nn.functional.pad = functional_pad - - torch.bmm = torch_bmm torch.fft.fftn = fft_fftn torch.fft.ifftn = fft_ifftn + if not device_supports_fp64: torch.from_numpy = from_numpy torch.as_tensor = as_tensor @@ -462,6 +365,18 @@ def ipex_hijacks(): except Exception: pass + if os.environ.get("IPEX_FORCE_ATTENTION_SLICE", "0") == "0": + if torch_version[0] > 2 or (torch_version[0] == 2 and torch_version[1] >= 7): + use_dynamic_attention = False # torch 2.7 has flash atten support + else: + use_dynamic_attention = True + else: + use_dynamic_attention = bool(os.environ.get("IPEX_FORCE_ATTENTION_SLICE", "0") == "1") + + if use_dynamic_attention: + from .attention import dynamic_scaled_dot_product_attention + torch.nn.functional.scaled_dot_product_attention = dynamic_scaled_dot_product_attention + # AMP: torch.amp.grad_scaler.GradScaler.__init__ = GradScaler_init torch.is_autocast_enabled = torch_is_autocast_enabled diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 578c96093..baebb62c2 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -13,7 +13,7 @@ from modules import shared, devices, errors, model_quant, sd_models, sd_models_c processor = None model = None loaded: str = None -quant_args = {} +quant_args = None vlm_default = "Alibaba Qwen 2.5 VL 4B" vlm_models = { "Google Gemma 3 4B": "google/gemma-3-4b-it", @@ -27,12 +27,18 @@ vlm_models = { "Apple FastVLM 0.5B": "apple/FastVLM-0.5B", "Apple FastVLM 1.5B": "apple/FastVLM-1.5B", "Apple FastVLM 7B": "apple/FastVLM-7B", - "Microsoft Florence 2 Base": "microsoft/Florence-2-base-ft", # 0.5GB - "Microsoft Florence 2 Large": "microsoft/Florence-2-large-ft", # 1.5GB - "MiaoshouAI PromptGen 1.5 Base": "MiaoshouAI/Florence-2-base-PromptGen-v1.5@c06a5f02cc6071a5d65ee5d294cf3732d3097540", # 1.1GB - "MiaoshouAI PromptGen 1.5 Large": "MiaoshouAI/Florence-2-large-PromptGen-v1.5@28a42440e39c9c32b83f7ae74ec2b3d1540404f0", # 3.3GB - "MiaoshouAI PromptGen 2.0 Base": "MiaoshouAI/Florence-2-base-PromptGen-v2.0", # 1.1GB - "MiaoshouAI PromptGen 2.0 Large": "MiaoshouAI/Florence-2-large-PromptGen-v2.0", # 3.3GB + # "Microsoft Florence 2 Base": "microsoft/Florence-2-base-ft", # 0.5GB + # "Microsoft Florence 2 Large": "microsoft/Florence-2-large-ft", # 1.5GB + "Microsoft Florence 2 Base": "florence-community/Florence-2-base-ft", # 0.5GB + "Microsoft Florence 2 Large": "florence-community/Florence-2-large-ft", # 1.5GB + #"MiaoshouAI PromptGen 1.5 Base": "MiaoshouAI/Florence-2-base-PromptGen-v1.5@c06a5f02cc6071a5d65ee5d294cf3732d3097540", # 1.1GB + #"MiaoshouAI PromptGen 1.5 Large": "MiaoshouAI/Florence-2-large-PromptGen-v1.5@28a42440e39c9c32b83f7ae74ec2b3d1540404f0", # 3.3GB + #"MiaoshouAI PromptGen 2.0 Base": "MiaoshouAI/Florence-2-base-PromptGen-v2.0", # 1.1GB + #"MiaoshouAI PromptGen 2.0 Large": "MiaoshouAI/Florence-2-large-PromptGen-v2.0", # 3.3GB + "MiaoshouAI PromptGen 1.5 Base": "Disty0/Florence-2-base-PromptGen-v1.5", # 0.5GB + "MiaoshouAI PromptGen 1.5 Large": "Disty0/Florence-2-large-PromptGen-v1.5", # 1.5GB + "MiaoshouAI PromptGen 2.0 Base": "Disty0/Florence-2-base-PromptGen-v2.0", # 0.5GB + "MiaoshouAI PromptGen 2.0 Large": "Disty0/Florence-2-large-PromptGen-v2.0", # 1.5GB "CogFlorence 2.0 Large": "thwri/CogFlorence-2-Large-Freeze", # 1.6GB "CogFlorence 2.2 Large": "thwri/CogFlorence-2.2-Large", # 1.6GB "Moondream 2": "vikhyatk/moondream2", # 3.7GB @@ -88,14 +94,19 @@ def b64(image): def clean(response, question): strip = ['---', '\r', '\t', '**', '"', '“', '”', 'Assistant:', 'Caption:', '<|im_end|>', ''] - if isinstance(response, dict): - if 'task' in response: - response = response['task'] + if isinstance(response, str): + response = response.strip() + elif isinstance(response, dict): if 'answer' in response: response = response['answer'] - response = json.dumps(response) - if isinstance(response, list): + elif 'task' in response: + response = response['task'] + else: + response = json.dumps(response) + elif isinstance(response, list): response = response[0] + else: + response = str(response) question = question.replace('<', '').replace('>', '').replace('_', ' ') if question in response: response = response.split(question, 1)[1] @@ -354,7 +365,6 @@ def smol(question: str, image: Image.Image, repo: str = None, system_prompt: str repo, cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, - _attn_implementation="eager", **quant_args, ) processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) @@ -524,11 +534,13 @@ def moondream(question: str, image: Image.Image, repo: str = None): def florence(question: str, image: Image.Image, repo: str = None, revision: str = None): global processor, model, loaded # pylint: disable=global-statement _get_imports = transformers.dynamic_module_utils.get_imports + def get_imports(f): R = _get_imports(f) if "flash_attn" in R: R.remove("flash_attn") # flash_attn is optional return R + revision = None if '@' in repo: repo, revision = repo.split('@') @@ -536,6 +548,7 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str shared.log.debug(f'Interrogate load: vlm="{repo}" path="{shared.opts.hfcache_dir}"') transformers.dynamic_module_utils.get_imports = get_imports model = None + """ model = transformers.AutoModelForCausalLM.from_pretrained( repo, trust_remote_code=True, @@ -544,6 +557,15 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str cache_dir=shared.opts.hfcache_dir, **quant_args, ) + """ + model = transformers.Florence2ForConditionalGeneration.from_pretrained( + repo, + dtype=torch.bfloat16, + revision=revision, + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + **quant_args, + ) processor = transformers.AutoProcessor.from_pretrained(repo, trust_remote_code=True, revision=revision, cache_dir=shared.opts.hfcache_dir) transformers.dynamic_module_utils.get_imports = _get_imports loaded = repo @@ -607,7 +629,8 @@ def interrogate(question:str='', system_prompt:str=None, prompt:str=None, image: global quant_args # pylint: disable=global-statement jobid = shared.state.begin('Interrogate LLM') t0 = time.time() - quant_args = model_quant.create_config(module='LLM') + if quant_args is None: + quant_args = model_quant.create_config(module='LLM') model_name = model_name or shared.opts.interrogate_vlm_model if isinstance(image, list): image = image[0] if len(image) > 0 else None @@ -623,9 +646,12 @@ def interrogate(question:str='', system_prompt:str=None, prompt:str=None, image: question = prompt if len(question) < 2: question = "Describe the image." + + """ if shared.sd_loaded: from modules.sd_models import apply_balanced_offload # prevent circular import apply_balanced_offload(shared.sd_model) + """ from modules import modelloader modelloader.hf_login() diff --git a/modules/ipadapter.py b/modules/ipadapter.py index 676045d1f..88a01e2cd 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -354,6 +354,18 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt adapter_scales[i] = 0.00 pipe.set_ip_adapter_scale(adapter_scales if len(adapter_scales) > 1 else adapter_scales[0]) ip_str = [f'{os.path.splitext(adapter)[0]}:{scale}:{start}:{end}:{crop}' for adapter, scale, start, end, crop in zip(adapter_names, adapter_scales, adapter_starts, adapter_ends, adapter_crops)] + if hasattr(pipe, 'transformer') and 'Nunchaku' in pipe.transformer.__class__.__name__: + if isinstance(repos, str): + sd_models.clear_caches(full=True) + import accelerate + accelerate.hooks.remove_hook_from_module(pipe.transformer, recurse=True) + pipe.transformer = pipe.transformer.to(devices.device) + from nunchaku.models.ip_adapter.diffusers_adapters import apply_IPA_on_pipe + apply_IPA_on_pipe(pipe, ip_adapter_scale=adapter_scales[0], repo_id=repos) + pipe = sd_models.apply_balanced_offload(pipe) + shared.log.debug(f'IP adapter load: engine=nunchaku scale={adapter_scales[0]} repo="{repos}"') + else: + shared.log.error('IP adapter: Nunchaku only supports single adapter') p.task_args['ip_adapter_image'] = crop_images(adapter_images, adapter_crops) if len(adapter_masks) > 0: p.cross_attention_kwargs = { 'ip_adapter_masks': adapter_masks } diff --git a/modules/loader.py b/modules/loader.py index efd58ff11..7ee969962 100644 --- a/modules/loader.py +++ b/modules/loader.py @@ -123,15 +123,20 @@ timer.startup.record("pydantic") import tqdm as tqdm_lib # pylint: disable=C0411 from tqdm.rich import tqdm # pylint: disable=W0611,C0411 -import diffusers.utils.import_utils # pylint: disable=W0611,C0411 -diffusers.utils.import_utils._k_diffusion_available = True # pylint: disable=protected-access # monkey-patch since we use k-diffusion from git -diffusers.utils.import_utils._k_diffusion_version = '0.0.12' # pylint: disable=protected-access +try: + import diffusers.utils.import_utils # pylint: disable=W0611,C0411 + diffusers.utils.import_utils._k_diffusion_available = True # pylint: disable=protected-access # monkey-patch since we use k-diffusion from git + diffusers.utils.import_utils._k_diffusion_version = '0.0.12' # pylint: disable=protected-access -import diffusers # pylint: disable=W0611,C0411 -import diffusers.loaders.single_file # pylint: disable=W0611,C0411 -diffusers.loaders.single_file.logging.tqdm = partial(tqdm, unit='C') -logging.getLogger("diffusers.loaders.single_file").setLevel(logging.ERROR) -timer.startup.record("diffusers") + import diffusers # pylint: disable=W0611,C0411 + import diffusers.loaders.single_file # pylint: disable=W0611,C0411 + diffusers.loaders.single_file.logging.tqdm = partial(tqdm, unit='C') + logging.getLogger("diffusers.loaders.single_file").setLevel(logging.ERROR) + timer.startup.record("diffusers") +except Exception as e: + errors.log.error(f'Loader: diffusers=={diffusers.__version__ if "diffusers" in sys.modules else None} {e}') + errors.log.error('Please restart re-run the installer') + sys.exit(1) import huggingface_hub # pylint: disable=W0611,C0411 timer.startup.record("hfhub") diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index d18444ea0..ee657d2d1 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -2,7 +2,7 @@ from typing import List import os import re import numpy as np -from modules.lora import networks, lora_overrides, lora_load +from modules.lora import networks, lora_overrides, lora_load, lora_diffusers from modules.lora import lora_common as l from modules import extra_networks, shared, sd_models @@ -83,21 +83,34 @@ def infotext(p): p.extra_generation_params["LoRA hashes"] = ", ".join(network_hashes) +def to_float(value): + try: + return float(value) + except (ValueError, TypeError): + return value + + def parse(p, params_list, step=0): names = [] te_multipliers = [] unet_multipliers = [] dyn_dims = [] + lora_modules = [] for params in params_list: - assert params.items names.append(params.positional[0]) - te_multiplier = params.named.get("te", params.positional[1] if len(params.positional) > 1 else shared.opts.extra_networks_default_multiplier) + + default_multiplier = params.positional[1] if len(params.positional) > 1 else shared.opts.extra_networks_default_multiplier + default_multiplier = to_float(default_multiplier) + if isinstance(default_multiplier, str) and "@" not in default_multiplier: + default_multiplier = shared.opts.extra_networks_default_multiplier + + te_multiplier = params.named.get("te", default_multiplier) if isinstance(te_multiplier, str) and "@" in te_multiplier: te_multiplier = get_stepwise(te_multiplier, step, p.steps) else: - te_multiplier = float(te_multiplier) - unet_multiplier = [params.positional[2] if len(params.positional) > 2 else te_multiplier] * 3 - unet_multiplier = [params.named.get("unet", unet_multiplier[0])] * 3 + te_multiplier = to_float(te_multiplier) + + unet_multiplier = 3 * [params.named.get("unet", te_multiplier)] # fill all 3 with same value unet_multiplier[0] = params.named.get("in", unet_multiplier[0]) unet_multiplier[1] = params.named.get("mid", unet_multiplier[1]) unet_multiplier[2] = params.named.get("out", unet_multiplier[2]) @@ -105,13 +118,23 @@ def parse(p, params_list, step=0): if isinstance(unet_multiplier[i], str) and "@" in unet_multiplier[i]: unet_multiplier[i] = get_stepwise(unet_multiplier[i], step, p.steps) else: - unet_multiplier[i] = float(unet_multiplier[i]) - dyn_dim = int(params.positional[3]) if len(params.positional) > 3 else None - dyn_dim = int(params.named["dyn"]) if "dyn" in params.named else dyn_dim + unet_multiplier[i] = to_float(unet_multiplier[i]) + + dyn_dim = int(params.named["dyn"]) if "dyn" in params.named else None te_multipliers.append(te_multiplier) unet_multipliers.append(unet_multiplier) dyn_dims.append(dyn_dim) - return names, te_multipliers, unet_multipliers, dyn_dims + + lora_module = [] + if 'high' in params.positional or 'HIGH 14B' in params.positional[0]: + lora_module.append('transformer') + if 'low' in params.positional or 'LOW 14B' in params.positional[0]: + lora_module.append('transformer_2') + if params.named.get('module', None) is not None: + lora_module.append(params.named['module'].lower()) + lora_modules.append(lora_module) + + return names, te_multipliers, unet_multipliers, dyn_dims, lora_modules def unload_diffusers(): @@ -168,7 +191,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): if len(params_list) > 0 and not self.active: # activate patches once self.active = True self.model = shared.opts.sd_model_checkpoint - names, te_multipliers, unet_multipliers, dyn_dims = parse(p, params_list, step) + names, te_multipliers, unet_multipliers, dyn_dims, lora_modules = parse(p, params_list, step) requested = self.signature(names, te_multipliers, unet_multipliers) load_method = lora_overrides.get_method() @@ -181,7 +204,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): has_changed = False # diffusers handles its own loading if len(exclude) == 0: jobid = shared.state.begin('LoRA') - lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load only on first call + lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims, lora_modules) # load only on first call sd_models.set_diffuser_offload(shared.sd_model, op="model") shared.state.end(jobid) elif load_method == 'nunchaku': @@ -208,7 +231,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): shared.log.info(f'Network load: type=LoRA apply={[n.name for n in l.loaded_networks]} method={load_method} mode={"fuse" if shared.opts.lora_fuse_diffusers else "backup"} te={te_multipliers} unet={unet_multipliers} time={l.timer.summary}') def deactivate(self, p): - if len(lora_load.diffuser_loaded) > 0: + if len(lora_diffusers.diffuser_loaded) > 0: if not (shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled is True): unload_diffusers() if self.active and l.debug: diff --git a/modules/lora/lora_diffusers.py b/modules/lora/lora_diffusers.py new file mode 100644 index 000000000..29402febe --- /dev/null +++ b/modules/lora/lora_diffusers.py @@ -0,0 +1,92 @@ +from typing import Union +import os +import time +import diffusers +from modules import shared, errors +from modules.lora import network +from modules.lora import lora_common as l + + +diffuser_loaded = [] +diffuser_scales = [] + + +def load_per_module(sd_model: diffusers.DiffusionPipeline, filename: str, adapter_name: str, lora_modules: list[str]): + shared.log.debug(f'LoRA load: modules={lora_modules}') + try: + state_dict = sd_model.lora_state_dict(filename) + if isinstance(state_dict, tuple) and len(state_dict) == 2: + state_dict, network_alphas = state_dict + else: + network_alphas = {} + except Exception as e: + shared.log.error(f'LoRA load: {e}') + if l.debug: + errors.display(e, "LoRA") + return None + for lora_module in lora_modules: + if lora_module == 'transformer': + if hasattr(sd_model, 'transformer') and sd_model.transformer is not None: + sd_model.load_lora_into_transformer(state_dict, transformer=sd_model.transformer, adapter_name=adapter_name) + else: + shared.log.warning(f'LoRA load: requested={lora_module} missing') + elif lora_module == 'transformer_2': + if hasattr(sd_model, 'transformer_2') and sd_model.transformer_2 is not None: + sd_model.load_lora_into_transformer(state_dict, transformer=sd_model.transformer_2, adapter_name=adapter_name) + else: + shared.log.warning(f'LoRA load: requested={lora_module} missing') + elif lora_module == 'unet': + if hasattr(sd_model, 'unet') and sd_model.unet is not None: + sd_model.load_lora_into_unet(state_dict, network_alphas, unet=sd_model.unet, adapter_name=adapter_name) + else: + shared.log.warning(f'LoRA load: requested={lora_module} missing') + elif lora_module == 'text_encoder' or lora_module == 'te': + if hasattr(sd_model, 'text_encoder') and sd_model.text_encoder is not None: + sd_model.load_lora_into_text_encoder(state_dict, network_alphas, text_encoder=sd_model.text_encoder, adapter_name=adapter_name) + else: + shared.log.warning(f'LoRA load: requested={lora_module} missing') + else: + shared.log.warning(f'LoRA load: requested={lora_module} unknown') + return adapter_name + + +def load_diffusers(name: str, network_on_disk: network.NetworkOnDisk, lora_scale:float=shared.opts.extra_networks_default_multiplier, lora_module=None) -> Union[network.Network, None]: + t0 = time.time() + name = name.replace(".", "_") + sd_model: diffusers.DiffusionPipeline = getattr(shared.sd_model, "pipe", shared.sd_model) + shared.log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_diffusers}') + if not hasattr(sd_model, 'load_lora_weights'): + shared.log.error(f'Network load: type=LoRA class={sd_model.__class__} does not implement load lora') + return None + try: + if lora_module is not None and isinstance(lora_module, list) and len(lora_module) > 0: + name = load_per_module(sd_model, network_on_disk.filename, adapter_name=name, lora_modules=lora_module) + else: + sd_model.load_lora_weights(network_on_disk.filename, adapter_name=name) + except Exception as e: + if 'already in use' in str(e): + pass + else: + if 'following keys have not been correctly renamed' in str(e): + shared.log.error(f'Network load: type=LoRA name="{name}" diffusers unsupported format') + elif 'object has no attribute' in str(e): + shared.log.error(f'Network load: type=LoRA name="{name}" diffusers empty module') + else: + shared.log.error(f'Network load: type=LoRA name="{name}" {e}') + if l.debug: + errors.display(e, "LoRA") + return None + if name is None: + return None + if name not in diffuser_loaded: + list_adapters = sd_model.get_list_adapters() + list_adapters = [adapter for adapters in list_adapters.values() for adapter in adapters] + if name not in list_adapters: + shared.log.error(f'Network load: type=LoRA name="{name}" adapters={list_adapters} not loaded') + else: + diffuser_loaded.append(name) + diffuser_scales.append(lora_scale) + net = network.Network(name, network_on_disk) + net.mtime = os.path.getmtime(network_on_disk.filename) + l.timer.activate += time.time() - t0 + return net diff --git a/modules/lora/lora_load.py b/modules/lora/lora_load.py index 0b74c6c6d..f629c2772 100644 --- a/modules/lora/lora_load.py +++ b/modules/lora/lora_load.py @@ -3,12 +3,10 @@ import os import time import concurrent from modules import shared, errors, sd_models, sd_models_compile, files_cache -from modules.lora import network, lora_overrides, lora_convert +from modules.lora import network, lora_overrides, lora_convert, lora_diffusers from modules.lora import lora_common as l -diffuser_loaded = [] -diffuser_scales = [] lora_cache = {} available_networks = {} available_network_aliases = {} @@ -17,41 +15,6 @@ available_network_hash_lookup = {} dump_lora_keys = os.environ.get('SD_LORA_DUMP', None) is not None -def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_default_multiplier) -> Union[network.Network, None]: - t0 = time.time() - name = name.replace(".", "_") - sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) - shared.log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_diffusers}') - if not hasattr(sd_model, 'load_lora_weights'): - shared.log.error(f'Network load: type=LoRA class={sd_model.__class__} does not implement load lora') - return None - try: - sd_model.load_lora_weights(network_on_disk.filename, adapter_name=name) - except Exception as e: - if 'already in use' in str(e): - pass - else: - if 'The following keys have not been correctly renamed' in str(e): - shared.log.error(f'Network load: type=LoRA name="{name}" diffusers unsupported format') - else: - shared.log.error(f'Network load: type=LoRA name="{name}" {e}') - if l.debug: - errors.display(e, "LoRA") - return None - if name not in diffuser_loaded: - list_adapters = sd_model.get_list_adapters() - list_adapters = [adapter for adapters in list_adapters.values() for adapter in adapters] - if name not in list_adapters: - shared.log.error(f'Network load: type=LoRA name="{name}" adapters={list_adapters} not loaded') - else: - diffuser_loaded.append(name) - diffuser_scales.append(lora_scale) - net = network.Network(name, network_on_disk) - net.mtime = os.path.getmtime(network_on_disk.filename) - l.timer.activate += time.time() - t0 - return net - - def lora_dump(lora, dct): import tempfile sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) @@ -73,7 +36,7 @@ def lora_dump(lora, dct): f.write(line + "\n") -def load_safetensors(name, network_on_disk) -> Union[network.Network, None]: +def load_safetensors(name, network_on_disk: network.NetworkOnDisk) -> Union[network.Network, None]: if not shared.sd_loaded: return None @@ -261,15 +224,15 @@ def gather_networks(names): return networks_on_disk -def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None): +def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None, lora_modules=None): networks_on_disk = gather_networks(names) failed_to_load_networks = [] recompile_model, skip_lora_load = maybe_recompile_model(names, te_multipliers) sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) l.loaded_networks.clear() - diffuser_loaded.clear() - diffuser_scales.clear() + lora_diffusers.diffuser_loaded.clear() + lora_diffusers.diffuser_scales.clear() t0 = time.time() for i, (network_on_disk, name) in enumerate(zip(networks_on_disk, names)): @@ -279,11 +242,13 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non if l.debug: shared.log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" hash="{shorthash}"') try: + lora_scale = te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier + lora_module = lora_modules[i] if lora_modules and len(lora_modules) > i else None if recompile_model: - shared.compiled_model_state.lora_model.append(f"{name}:{te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier}") + shared.compiled_model_state.lora_model.append(f"{name}:{lora_scale}") lora_method = lora_overrides.get_method(shorthash) if lora_method == 'diffusers': - net = load_diffusers(name, network_on_disk, lora_scale=te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier) + net = lora_diffusers.load_diffusers(name, network_on_disk, lora_scale, lora_module) elif lora_method == 'nunchaku': pass # handled directly from extra_networks_lora.load_nunchaku else: @@ -311,21 +276,21 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non name = next(iter(lora_cache)) lora_cache.pop(name, None) - if not skip_lora_load and len(diffuser_loaded) > 0: - shared.log.debug(f'Network load: type=LoRA loaded={diffuser_loaded} available={sd_model.get_list_adapters()} active={sd_model.get_active_adapters()} scales={diffuser_scales}') + if not skip_lora_load and len(lora_diffusers.diffuser_loaded) > 0: + shared.log.debug(f'Network load: type=LoRA loaded={lora_diffusers.diffuser_loaded} available={sd_model.get_list_adapters()} active={sd_model.get_active_adapters()} scales={lora_diffusers.diffuser_scales}') try: t1 = time.time() if l.debug: shared.log.trace(f'Network load: type=LoRA list={sd_model.get_list_adapters()}') shared.log.trace(f'Network load: type=LoRA active={sd_model.get_active_adapters()}') - sd_model.set_adapters(adapter_names=diffuser_loaded, adapter_weights=diffuser_scales) + sd_model.set_adapters(adapter_names=lora_diffusers.diffuser_loaded, adapter_weights=lora_diffusers.diffuser_scales) except Exception as e: shared.log.error(f'Network load: type=LoRA action=set {e}') if l.debug: errors.display(e, 'LoRA') try: if shared.opts.lora_fuse_diffusers and not lora_overrides.disable_fuse(): - sd_model.fuse_lora(adapter_names=diffuser_loaded, lora_scale=1.0, fuse_unet=True, fuse_text_encoder=True) # diffusers with fuse uses fixed scale since later apply does the scaling + sd_model.fuse_lora(adapter_names=lora_diffusers.diffuser_loaded, lora_scale=1.0, fuse_unet=True, fuse_text_encoder=True) # diffusers with fuse uses fixed scale since later apply does the scaling sd_model.unload_lora_weights() l.timer.activate += time.time() - t1 except Exception as e: diff --git a/modules/lora/lora_nunchaku.py b/modules/lora/lora_nunchaku.py index 63a71b46f..de4773158 100644 --- a/modules/lora/lora_nunchaku.py +++ b/modules/lora/lora_nunchaku.py @@ -15,6 +15,9 @@ def load_nunchaku(names, strengths): is_changed = loras != previously_loaded if not is_changed: return False + if not hasattr(shared.sd_model, 'transformer') or not hasattr(shared.sd_model.transformer, 'update_lora_params'): + shared.log.error(f'Network load: type=LoRA method=nunchaku model={shared.sd_model.__class__.__name__} unsupported') + return False previously_loaded = loras try: @@ -27,7 +30,7 @@ def load_nunchaku(names, strengths): lora_common.timer.load = t1 - t0 shared.log.debug(f"Network load: type=LoRA method=nunchaku loras={names} strength={strengths} time={t1-t0:.3f}") except Exception as e: - shared.log.errors(f'Network load: type=LoRA method=nunchaku {e}') + shared.log.error(f'Network load: type=LoRA method=nunchaku {e}') if lora_common.debug: errors.display(e, 'LoRA') return is_changed diff --git a/modules/lora/lora_overrides.py b/modules/lora/lora_overrides.py index 61f2dad46..5889e1dec 100644 --- a/modules/lora/lora_overrides.py +++ b/modules/lora/lora_overrides.py @@ -61,7 +61,9 @@ def get_method(shorthash=''): use_diffusers = use_diffusers or any(x.startswith(shorthash) for x in maybe_diffusers) if shared.opts.lora_force_diffusers and len(shorthash) > 4: use_diffusers = use_diffusers or any(x.startswith(shorthash) for x in force_diffusers) - use_nunchaku = hasattr(shared.sd_model, 'transformer') and 'Nunchaku' in shared.sd_model.transformer.__class__.__name__ + nunchaku_dit = hasattr(shared.sd_model, 'transformer') and 'Nunchaku' in shared.sd_model.transformer.__class__.__name__ + nunchaku_unet = hasattr(shared.sd_model, 'unet') and 'Nunchaku' in shared.sd_model.unet.__class__.__name__ + use_nunchaku = nunchaku_dit or nunchaku_unet if use_nunchaku: return 'nunchaku' elif use_diffusers: diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index 27b033af5..e0626bbe4 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -3,17 +3,34 @@ import gradio as gr from modules import shared, ui_sections, ui_symbols, ui_common from modules.ui_components import ToolButton from modules.video_models.video_utils import get_codecs +from modules.video_models.models_def import models from modules.ltx import ltx_process debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None +def load_model(model_name): + if model_name is None or model_name == 'None': + shared.log.info('LTX model unload') + from modules import sd_models + sd_models.unload_model_weights() + return + else: + model_instance = [m for m in models['LTX Video'] if m.name == model_name][0] + from modules.video_models import video_load + video_load.load_model(model_instance) + + def create_ui(prompt, negative, styles, overrides): with gr.Row(): with gr.Column(variant='compact', elem_id="ltx_settings", elem_classes=['settings-column'], scale=1): with gr.Row(): generate = gr.Button('Generate', elem_id="ltx_generate_btn", variant='primary', visible=False) + with gr.Row(): + ltx_models = [m.name for m in models['LTX Video']] + model = gr.Dropdown(label='LTX model', choices=ltx_models, value=ltx_models[0]) + model.change(fn=load_model, inputs=[model], outputs=[], show_progress=True) with gr.Accordion(open=True, label="LTX size", elem_id='ltx_generate_accordion'): with gr.Row(): width, height = ui_sections.create_resolution_inputs('ltx', default_width=832, default_height=480) diff --git a/modules/memstats.py b/modules/memstats.py index fdc94df37..68a7d27ea 100644 --- a/modules/memstats.py +++ b/modules/memstats.py @@ -54,11 +54,28 @@ def ram_stats(): ram_total = 100 * res.rss / process.memory_percent() ram_total = min(ram_total, get_docker_limit(), get_runpod_limit()) ram['total'] = gb(ram_total) - ram['used'] = gb(res.rss) - ram['free'] = round(ram['total'] - ram['used']) + ram['rss'] = gb(res.rss) except Exception as e: ram['total'] = 0 + ram['rss'] = 0 + ram['error'] = str(e) + if not fail_once: + shared.log.error(f'RAM stats: {e}') + errors.display(e, 'RAM stats') + fail_once = True + try: + vmem = psutil.virtual_memory() + ram['used'] = gb(vmem.used) if hasattr(vmem, 'used') else 0 + ram['free'] = gb(vmem.free) if hasattr(vmem, 'free') else 0 + ram['avail'] = gb(vmem.available) if hasattr(vmem, 'available') else 0 + ram['buffers'] = gb(vmem.buffers) if hasattr(vmem, 'buffers') else 0 + ram['cached'] = gb(vmem.cached) if hasattr(vmem, 'cached') else 0 + except Exception as e: ram['used'] = 0 + ram['free'] = 0 + ram['avail'] = 0 + ram['buffers'] = 0 + ram['cached'] = 0 ram['error'] = str(e) if not fail_once: shared.log.error(f'RAM stats: {e}') diff --git a/modules/model_quant.py b/modules/model_quant.py index 33495b327..19b7ae7a7 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -501,11 +501,6 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc: bool = True, weigh modules_to_not_convert.extend(model._skip_layerwise_casting_patterns) # pylint: disable=protected-access if model.__class__.__name__ == "ChromaTransformer2DModel": modules_to_not_convert.append("distilled_guidance_layer") - if model.__class__.__name__ == "QwenImageTransformer2DModel": - if "minimum_6bit" not in modules_dtype_dict.keys(): - modules_dtype_dict["minimum_6bit"] = ["img_mod", "pos_embed", "time_text_embed", "img_in", "txt_in", "norm_out"] - else: - modules_dtype_dict["minimum_6bit"].extend(["img_mod", "pos_embed", "time_text_embed", "img_in", "txt_in", "norm_out"]) sdnq_modules_to_not_convert = [m.strip() for m in re.split(';|,| ', shared.opts.sdnq_modules_to_not_convert) if len(m.strip()) > 1] if len(sdnq_modules_to_not_convert) > 0: diff --git a/modules/modular.py b/modules/modular.py new file mode 100644 index 000000000..5f3bfefc2 --- /dev/null +++ b/modules/modular.py @@ -0,0 +1,64 @@ +import time +import diffusers +from modules import shared + + +modular_map= { + 'StableDiffusionXLPipeline': 'StableDiffusionXLAutoBlocks', + 'StableDiffusionXLImg2ImgPipeline': 'StableDiffusionXLAutoBlocks', + 'StableDiffusionXLInpaintPipeline': 'StableDiffusionXLAutoBlocks', + 'FluxPipeline': 'FluxAutoBlocks', + 'FluxImg2ImgPipeline': 'FluxAutoBlocks', + 'FluxInpaintPipeline': 'FluxAutoBlocks', + 'WanPipeline': 'WanAutoBlocks', + 'WanImageToVideoPipeline': 'WanAutoBlocks', + 'QwenImagePipeline': 'QwenImageAutoBlocks', + 'QwenImageEditPipeline': 'QwenImageEditAutoBlocks', +} + + +def is_compatible(diffusion_pipeline: diffusers.DiffusionPipeline) -> bool: + if not shared.opts.model_modular_enable: + return False + compatible = diffusion_pipeline.__class__.__name__ in modular_map + if not compatible: + shared.log.debug(f'Modular: source={diffusion_pipeline.__class__.__name__} incompatible pipeline') + return compatible + + +def convert_to_modular(diffusion_pipeline: diffusers.DiffusionPipeline) -> diffusers.ModularPipeline: + modular_pipe = None + try: + t0 = time.time() + modular_cls = modular_map.get(diffusion_pipeline.__class__.__name__, None) + if modular_cls is None: + raise ValueError(f'unknown: cls={diffusion_pipeline.__class__.__name__}') + modular_cls = getattr(diffusers, modular_cls, None) + if modular_cls is None: + raise ValueError(f'invalid: cls={diffusion_pipeline.__class__.__name__}') + modular_blocks = modular_cls() + modular_pipe = modular_blocks.init_pipeline() + components_dct = {k: v for k, v in diffusion_pipeline.components.items() if v is not None} + modular_pipe.update_components(**components_dct, **diffusion_pipeline.parameters) + modular_pipe.original_pipe = diffusion_pipeline + t1 = time.time() + shared.log.debug(f'Modular: source={diffusion_pipeline.__class__.__name__} target={modular_pipe.__class__.__name__} time={t1 - t0:.2f}') + """ + for expected_input_param in modular_pipe.blocks.inputs: + name = expected_input_param.name + default = expected_input_param.default + kwargs_type = expected_input_param.kwargs_type + shared.log.trace(f'Modular input: name={name} type={kwargs_type} default={default}') + """ + + except Exception as e: + shared.log.error(f'Modular: {e}') + raise e + return modular_pipe + + +def restore_standard(modular_pipe): + if hasattr(modular_pipe, 'original_pipe'): + shared.log.debug(f'Modular: source={modular_pipe.__class__.__name__} target={modular_pipe.original_pipe.__class__.__name__}') + return modular_pipe.original_pipe + return modular_pipe diff --git a/modules/modular_guiders.py b/modules/modular_guiders.py new file mode 100644 index 000000000..556ad52b4 --- /dev/null +++ b/modules/modular_guiders.py @@ -0,0 +1,87 @@ +import diffusers +from modules import shared, errors, processing + + +# ['Default', 'CFG', 'Zero', 'PAG', 'APG', 'SLG', 'SEG', 'TCFG', 'FDG'] +guiders = { + # 'None': { 'cls': None, 'args': {}, }, + 'Default': { 'cls': None, 'args': {}, }, + 'CFG: ClassifierFreeGuidance': { 'cls': diffusers.ClassifierFreeGuidance, 'args': {} }, + 'Auto: AutoGuidance': { 'cls': diffusers.AutoGuidance, 'args': { 'dropout': 1.0, 'auto_guidance_layers': [7, 8, 9], 'auto_guidance_config': None } }, + 'Zero: ClassifierFreeZeroStar': { 'cls': diffusers.ClassifierFreeZeroStarGuidance, 'args': { 'zero_init_steps': 1 } }, + 'PAG: PerturbedAttentionGuidance': { 'cls': diffusers.PerturbedAttentionGuidance, 'args': { 'perturbed_guidance_scale': 2.8, 'perturbed_guidance_start': 0.01, 'perturbed_guidance_stop': 0.2, 'perturbed_guidance_layers': [7, 8, 9], 'perturbed_guidance_config': None } }, + 'APG: AdaptiveProjectedGuidance': { 'cls': diffusers.AdaptiveProjectedGuidance, 'args': { 'adaptive_projected_guidance_momentum': -1, 'adaptive_projected_guidance_rescale': 15.0 } }, + 'SLG: SkipLayerGuidance': { 'cls': diffusers.SkipLayerGuidance, 'args': { 'skip_layer_guidance_scale': 2.8, 'skip_layer_guidance_start': 0.01, 'skip_layer_guidance_stop': 0.2, 'skip_layer_guidance_layers': [7, 8, 9], 'skip_layer_config': None } }, + 'SEG: SmoothedEnergyGuidance': { 'cls': diffusers.SmoothedEnergyGuidance, 'args': { 'seg_guidance_scale': 3.0, 'seg_blur_sigma': 9999999.0, 'seg_blur_threshold_inf': 9999.0, 'seg_guidance_start': 0.0, 'seg_guidance_stop': 1.0, 'seg_guidance_layers': [7, 8, 9], 'seg_guidance_config': None } }, + 'TCFG: TangentialClassifierFreeGuidance': { 'cls': diffusers.TangentialClassifierFreeGuidance, 'args': {} }, + 'FDG: FrequencyDecoupledGuidance': { 'cls': diffusers.FrequencyDecoupledGuidance, 'args': { 'guidance_scales': [10.0, 5.0], 'parallel_weights': 1.0, 'guidance_rescale_space': "data" } }, +} +base_args = { + 'guidance_scale': 6.0, + 'guidance_rescale': 0.0, + 'start': 0.0, + 'stop': 1.0, +} + + +def set_guider(p: processing.StableDiffusionProcessing): + guidance_name = p.guidance_name or 'Default' + if guidance_name not in guiders: + return + + if guidance_name == 'Default': + if hasattr(shared.sd_model, 'default_guider'): + guider_info = shared.sd_model.default_guider + shared.sd_model.update_components(guider=guider_info) + else: + guider_info = shared.sd_model.get_component_spec("guider") + shared.sd_model.default_guider = guider_info + guider_cls = guider_info.type_hint + if guider_info is not None and guider_cls is not None and guider_info.config is not None: + guider_args = {k: v for k, v in guider_info.config.items() if not k.startswith('_') and v is not None} + else: + guider_args = {} + shared.log.info(f'Guider: name={guidance_name} cls={guider_cls.__name__} args={guider_args}') + return + if guidance_name == 'None': + shared.sd_model.update_components(guider=None) # breaks the pipeline + shared.log.info(f'Guider: name={guidance_name}') + return + + guider_info = guiders[guidance_name] + guider_cls = guider_info['cls'] + guider_args = {} + for k, v in base_args.items(): + if v is not None and v >= 0.0: + guider_args[k] = v + shared.log.warning('Guiders: partially implemented') # TODO: guiders + for k, v in guider_info['args'].items(): + try: + if k is None: + pass + elif k.endswith('_layers') and isinstance(v, str): + guider_args[k] = [int(x.strip()) for x in v.split(',') if x.strip().isdigit()] + elif k.endswith('_config'): + # if lsc_enabled + # guider_args[k] = diffusers.LayerSkipConfig(...) + pass + elif isinstance(v, list) and len(v) > 0: + guider_args[k] = v + elif isinstance(v, int) and (v >= 0): + guider_args[k] = int(v) + elif isinstance(v, float) and (v >= 0.0): + guider_args[k] = float(v) + elif isinstance(v, str) and (len(v) > 0): + guider_args[k] = v + except Exception as e: + shared.log.error(f'Guiders: arg={k} value={v} error={e}') + errors.display(e, 'Guiders') + # guider_args.update(guider_info['args']) + if guider_cls is not None: + try: + guider_instance = guider_cls(**guider_args) + shared.log.info(f'Guider: name={guidance_name} cls={guider_cls.__name__} args={guider_args}') + shared.sd_model.update_components(guider=guider_instance) + except Exception as e: + shared.log.error(f'Guider: name={guidance_name} cls={guider_cls.__name__} args={guider_args} {e}') + return diff --git a/modules/processing.py b/modules/processing.py index 31378fa4a..d42929b23 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -130,7 +130,7 @@ def get_processed(*args, **kwargs): def process_images(p: StableDiffusionProcessing) -> Processed: timer.process.reset() - debug(f'Process images: {vars(p)}') + debug(f'Process images: class={p.__class__.__name__} {vars(p)}') if not hasattr(p.sd_model, 'sd_checkpoint_info'): shared.log.error('Processing: incomplete model') return None @@ -427,13 +427,20 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: for script_image, script_infotext in zip(results.images, results.infotexts): output_images.append(script_image) infotexts.append(script_infotext) + if samples is None: from modules.processing_diffusers import process_diffusers samples = process_diffusers(p) timer.process.record('process') - if not shared.opts.keep_incomplete and shared.state.interrupted: - samples = [] + if shared.state.interrupted: + shared.log.debug(f'Process: batch={n+1}/{p.n_iter} interrupted') + p.do_not_save_samples = not shared.opts.keep_incomplete + if shared.state.current_image is not None and isinstance(shared.state.current_image, Image.Image): + samples = [shared.state.current_image] + infotexts = [create_infotext(p, p.all_prompts, p.all_seeds, p.all_subseeds, index=0)] + else: + samples = [] if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner): p.scripts.postprocess_batch(p, samples, batch_number=n) diff --git a/modules/processing_args.py b/modules/processing_args.py index 5d40d8db0..c1f0b9cdb 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -18,6 +18,26 @@ debug_log = shared.log.trace if debug_enabled else lambda *args, **kwargs: None disable_pbar = os.environ.get('SD_DISABLE_PBAR', None) is not None +def task_modular_kwargs(p, model): # pylint: disable=unused-argument + # model_cls = model.__class__.__name__ + task_args = {} + p.ops.append('modular') + + processing_helpers.resize_init_images(p) + task_args['width'] = p.width + task_args['height'] = p.height + if len(getattr(p, 'init_images', [])) > 0: + task_args['image'] = p.init_images + task_args['strength'] = p.denoising_strength + mask_image = p.task_args.get('image_mask', None) or getattr(p, 'image_mask', None) or getattr(p, 'mask', None) + if mask_image is not None: + task_args['mask_image'] = mask_image + + if debug_enabled: + debug_log(f'Process task specific args: {task_args}') + return task_args + + def task_specific_kwargs(p, model): model_cls = model.__class__.__name__ vae_scale_factor = sd_vae.get_vae_scale_factor(model) @@ -103,9 +123,9 @@ def task_specific_kwargs(p, model): } # model specific args - if model_cls == 'QwenImageEditPipeline' and len(getattr(p, 'init_images', [])) == 0: + if 'QwenImageEdit' in model_cls and len(getattr(p, 'init_images', [])) == 0: task_args['image'] = [Image.new('RGB', (p.width, p.height), (0, 0, 0))] # monkey-patch so qwen-image-edit pipeline does not error-out on t2i - if model_cls == 'LatentConsistencyModelPipeline' and hasattr(p, 'init_images') and len(p.init_images) > 0: + if 'LatentConsistencyModelPipeline' in model_cls and hasattr(p, 'init_images') and len(p.init_images) > 0: p.ops.append('lcm') init_latents = [processing_vae.vae_encode(image, model=shared.sd_model, vae_type=p.vae_type).squeeze(dim=0) for image in p.init_images] init_latent = torch.stack(init_latents, dim=0).to(shared.device) @@ -116,7 +136,7 @@ def task_specific_kwargs(p, model): 'width': p.width if hasattr(p, 'width') else None, 'height': p.height if hasattr(p, 'height') else None, } - if model_cls == 'BlipDiffusionPipeline': + if 'BlipDiffusionPipeline' in model_cls: if len(getattr(p, 'init_images', [])) == 0: shared.log.error('BLiP diffusion requires init image') return task_args @@ -126,14 +146,26 @@ def task_specific_kwargs(p, model): 'target_subject_category': getattr(p, 'prompt', '').split()[-1], 'output_type': 'pil', } - if model.__class__.__name__ == 'WanImageToVideoPipeline' and hasattr(p, 'init_images') and len(p.init_images) > 0: + if ('WanImageToVideoPipeline' in model_cls) and (getattr(p, 'init_images', None) is not None) and (len(p.init_images) > 0): task_args['image'] = p.init_images[0] + if ('WanVACEPipeline' in model_cls) and (getattr(p, 'init_images', None) is not None) and (len(p.init_images) > 0): + task_args['reference_images'] = p.init_images if debug_enabled: debug_log(f'Process task specific args: {task_args}') return task_args +def get_params(model): + if hasattr(model, 'blocks') and hasattr(model.blocks, 'inputs'): # modular pipeline + possible = [input_param.name for input_param in model.blocks.inputs] + return possible + else: + signature = inspect.signature(type(model).__call__, follow_wrapped=True) + possible = list(signature.parameters) + return possible + + def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:typing.Optional[list]=None, negative_prompts_2:typing.Optional[list]=None, prompt_attention:typing.Optional[str]=None, desc:typing.Optional[str]='', **kwargs): t0 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) @@ -149,8 +181,8 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba', disable=disable_pbar) else: model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba') - signature = inspect.signature(type(model).__call__, follow_wrapped=True) - possible = list(signature.parameters) + + possible = get_params(model) if debug_enabled: debug_log(f'Process pipeline possible: {possible}') @@ -307,6 +339,14 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t args['control_strength'] = p.denoising_strength args['width'] = p.width args['height'] = p.height + if 'WanVACEPipeline' in model.__class__.__name__: + if isinstance(args['prompt'], list): + args['prompt'] = args['prompt'][0] if len(args['prompt']) > 0 else '' + if isinstance(args.get('negative_prompt', None), list): + args['negative_prompt'] = args['negative_prompt'][0] if len(args['negative_prompt']) > 0 else '' + if isinstance(args['generator'], list) and len(args['generator']) > 0: + args['generator'] = args['generator'][0] + # set callbacks if 'prior_callback_steps' in possible: # Wuerstchen / Cascade args['prior_callback_steps'] = 1 @@ -347,7 +387,11 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t args[arg] = kwargs[arg] # handle task specific args - task_kwargs = task_specific_kwargs(p, model) + if sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.MODULAR: + task_kwargs = task_modular_kwargs(p, model) + else: + task_kwargs = task_specific_kwargs(p, model) + pipe_args = getattr(p, 'task_args', {}) model_args = getattr(model, 'task_args', {}) task_kwargs.update(pipe_args or {}) @@ -397,8 +441,9 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t # handle implicit controlnet if 'control_image' in possible and 'control_image' not in args and 'image' in args: - debug_log('Process: set control image') - args['control_image'] = args['image'] + if sd_models.get_diffusers_task(model) != sd_models.DiffusersTaskType.MODULAR: + debug_log('Process: set control image') + args['control_image'] = args['image'] sd_hijack_hypertile.hypertile_set(p, hr=len(getattr(p, 'init_images', [])) > 0) diff --git a/modules/processing_class.py b/modules/processing_class.py index 7ea9c2eab..ddad4686e 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -36,7 +36,13 @@ class StableDiffusionProcessing: sampler_name: str = None, hr_sampler_name: str = None, eta: float = None, - # guidance + # modular guidance + guidance_name: str = 'Default', + guidance_scale: float = 6.0, + guidance_rescale: float = 0.0, + guidance_start: float = 0.0, + guidance_stop: float = 1.0, + # legacy guidance cfg_scale: float = 6.0, cfg_end: float = 1, diffusers_guidance_rescale: float = 0.0, @@ -247,6 +253,11 @@ class StableDiffusionProcessing: self.do_not_save_grid = do_not_save_grid self.override_settings_restore_afterwards = override_settings_restore_afterwards self.eta = eta + self.guidance_name = guidance_name + self.guidance_scale = guidance_scale + self.guidance_rescale = guidance_rescale + self.guidance_start = guidance_start + self.guidance_stop = guidance_stop self.cfg_scale = cfg_scale self.cfg_end = cfg_end self.diffusers_guidance_rescale = diffusers_guidance_rescale @@ -445,14 +456,14 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.hr_upscale_to_y = int(self.height * scale) else: if self.hr_resize_y == 0: - self.hr_upscale_to_x = self.hr_resize_x - self.hr_upscale_to_y = self.hr_resize_x * self.height // self.width + self.hr_upscale_to_x = int(self.hr_resize_x) + self.hr_upscale_to_y = int(self.hr_resize_x * self.height // self.width) elif self.hr_resize_x == 0: - self.hr_upscale_to_x = self.hr_resize_y * self.width // self.height - self.hr_upscale_to_y = self.hr_resize_y + self.hr_upscale_to_x = int(self.hr_resize_y * self.width // self.height) + self.hr_upscale_to_y = int(self.hr_resize_y) elif self.hr_resize_x > 0 and self.hr_resize_y > 0: - self.hr_upscale_to_x = self.hr_resize_x - self.hr_upscale_to_y = self.hr_resize_y + self.hr_upscale_to_x = int(self.hr_resize_x) + self.hr_upscale_to_y = int(self.hr_resize_y) shared.log.debug(f'Init hires: upscaler="{self.hr_upscaler}" sampler="{self.hr_sampler_name}" resize={self.hr_resize_x}x{self.hr_resize_y} upscale={self.hr_upscale_to_x}x{self.hr_upscale_to_y}') @@ -572,9 +583,9 @@ class StableDiffusionProcessingControl(StableDiffusionProcessingImg2Img): self.hr_force = force if use_scale: vae_scale_factor = sd_vae.get_vae_scale_factor() - self.hr_upscale_to_x, self.hr_upscale_to_y = vae_scale_factor * int(self.width * scale / vae_scale_factor), vae_scale_factor * int(self.height * scale / vae_scale_factor) + self.hr_upscale_to_x, self.hr_upscale_to_y = int(vae_scale_factor * int(self.width * scale / vae_scale_factor)), int(vae_scale_factor * int(self.height * scale / vae_scale_factor)) else: - self.hr_upscale_to_x, self.hr_upscale_to_y = self.hr_resize_x, self.hr_resize_y + self.hr_upscale_to_x, self.hr_upscale_to_y = int(self.hr_resize_x), int(self.hr_resize_y) def switch_class(p: StableDiffusionProcessing, new_class: type, dct: dict = None): diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 8353be172..277b5e1c5 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -67,7 +67,7 @@ def restore_state(p: processing.StableDiffusionProcessing): def process_pre(p: processing.StableDiffusionProcessing): - from modules import ipadapter, sd_hijack_freeu, para_attention, teacache, hidiffusion, ras, pag, cfgzero, transformer_cache, token_merge, linfusion + from modules import ipadapter, sd_hijack_freeu, para_attention, teacache, hidiffusion, ras, pag, cfgzero, transformer_cache, token_merge, linfusion, cachedit shared.log.info('Processing modifiers: apply') try: @@ -80,6 +80,7 @@ def process_pre(p: processing.StableDiffusionProcessing): pag.apply(p) cfgzero.apply(p) linfusion.apply(shared.sd_model) + cachedit.apply_cache_dit(shared.sd_model) # apply-only sd_hijack_freeu.apply_freeu(p) @@ -95,11 +96,20 @@ def process_pre(p: processing.StableDiffusionProcessing): # sd_models.move_model(shared.sd_model.unet, devices.device) # if hasattr(shared.sd_model, 'transformer'): # sd_models.move_model(shared.sd_model.transformer, devices.device) + + from modules import modular + if modular.is_compatible(shared.sd_model): + modular_pipe = modular.convert_to_modular(shared.sd_model) + if modular_pipe is not None: + shared.sd_model = modular_pipe + from modules import modular_guiders + modular_guiders.set_guider(p) + timer.process.record('pre') def process_post(p: processing.StableDiffusionProcessing): - from modules import ipadapter, hidiffusion, ras, pag, cfgzero, token_merge, linfusion + from modules import ipadapter, hidiffusion, ras, pag, cfgzero, token_merge, linfusion, cachedit shared.log.info('Processing modifiers: unapply') try: @@ -111,6 +121,7 @@ def process_post(p: processing.StableDiffusionProcessing): pag.unapply() cfgzero.unapply() linfusion.unapply(shared.sd_model) + cachedit.unapply_cache_dir(shared.sd_model) except Exception as e: shared.log.error(f'Processing unapply: {e}') errors.display(e, 'unapply') @@ -229,8 +240,8 @@ def process_hires(p: processing.StableDiffusionProcessing, output): p.hr_upscaler = p.resize_name p.hr_resize_mode = p.resize_mode p.hr_resize_context = p.resize_context - p.hr_upscale_to_x = p.width * p.hr_scale if p.hr_resize_x == 0 else p.hr_resize_x - p.hr_upscale_to_y = p.height * p.hr_scale if p.hr_resize_y == 0 else p.hr_resize_y + p.hr_upscale_to_x = int(p.width * p.hr_scale) if p.hr_resize_x == 0 else p.hr_resize_x + p.hr_upscale_to_y = int(p.height * p.hr_scale) if p.hr_resize_y == 0 else p.hr_resize_y # hires runs on original pipeline if hasattr(shared.sd_model, 'restore_pipeline') and (shared.sd_model.restore_pipeline is not None) and (not shared.opts.control_hires): @@ -487,7 +498,7 @@ def validate_pipeline(p: processing.StableDiffusionProcessing): if m.repo_cls is not None: models_cls.append(m.repo_cls.__name__) is_video_model = shared.sd_model.__class__.__name__ in models_cls - override_video_pipelines = ['WanPipeline', 'WanImageToVideoPipeline'] + override_video_pipelines = ['WanPipeline', 'WanImageToVideoPipeline', 'WanVACEPipeline'] is_video_pipeline = ('video' in p.__class__.__name__.lower()) or (shared.sd_model.__class__.__name__ in override_video_pipelines) if is_video_model and not is_video_pipeline: shared.log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} video model with non-video pipeline') diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index 976ab0c32..b58f0594a 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -9,6 +9,7 @@ import cv2 from PIL import Image from blendmodes.blend import blendLayers, BlendType from modules import shared, devices, images, sd_models, sd_samplers, sd_vae, sd_hijack_hypertile, processing_vae, timer +from modules.api import helpers debug = shared.log.trace if os.environ.get('SD_PROCESS_DEBUG', None) is not None else lambda *args, **kwargs: None @@ -16,6 +17,10 @@ debug_steps = shared.log.trace if os.environ.get('SD_STEPS_DEBUG', None) is not debug_steps('Trace: STEPS') +def is_modular(): + return sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.MODULAR + + def is_txt2img(): return sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE @@ -275,10 +280,38 @@ def validate_sample(tensor): return cast +def decode_images(image): + if isinstance(image, list): + decoded = [] + for i, img in enumerate(image): + if isinstance(img, str): + try: + decoded.append(helpers.decode_base64_to_image(img, quiet=True)) + except Exception as e: + shared.log.error(f'Decode image[{i}]: {e}') + elif isinstance(img, Image.Image): + decoded.append(img) + else: + shared.log.error(f'Decode image[{i}]: {type(img)} unknown type') + return decoded + elif isinstance(image, str): + try: + return helpers.decode_base64_to_image(image, quiet=True) + except Exception as e: + shared.log.error(f'Decode image: {e}') + elif isinstance(image, Image.Image): + return image + else: + shared.log.error(f'Decode image: {type(image)} unknown type') + return None + + def resize_init_images(p): if getattr(p, 'image', None) is not None and getattr(p, 'init_images', None) is None: p.init_images = [p.image] + if getattr(p, 'init_images', None) is not None and len(p.init_images) > 0: + p.init_images = decode_images(p.init_images) vae_scale_factor = sd_vae.get_vae_scale_factor() tgt_width, tgt_height = vae_scale_factor * math.ceil(p.init_images[0].width / vae_scale_factor), vae_scale_factor * math.ceil(p.init_images[0].height / vae_scale_factor) if p.init_images[0].size != (tgt_width, tgt_height): @@ -287,11 +320,14 @@ def resize_init_images(p): p.height = tgt_height p.width = tgt_width sd_hijack_hypertile.hypertile_set(p) - if getattr(p, 'mask', None) is not None and p.mask.size != (tgt_width, tgt_height): + if getattr(p, 'mask', None) is not None and p.mask is not None and p.mask.size != (tgt_width, tgt_height): + p.mask = decode_images(p.mask) p.mask = images.resize_image(1, p.mask, tgt_width, tgt_height, upscaler_name=None) - if getattr(p, 'init_mask', None) is not None and p.init_mask.size != (tgt_width, tgt_height): + if getattr(p, 'init_mask', None) is not None and p.init_mask is not None and p.init_mask.size != (tgt_width, tgt_height): + p.init_mask = decode_images(p.init_mask) p.init_mask = images.resize_image(1, p.init_mask, tgt_width, tgt_height, upscaler_name=None) - if getattr(p, 'mask_for_overlay', None) is not None and p.mask_for_overlay.size != (tgt_width, tgt_height): + if getattr(p, 'mask_for_overlay', None) is not None and p.mask_for_overlay is not None and p.mask_for_overlay.size != (tgt_width, tgt_height): + p.mask_for_overlay = decode_images(p.mask_for_overlay) p.mask_for_overlay = images.resize_image(1, p.mask_for_overlay, tgt_width, tgt_height, upscaler_name=None) return tgt_width, tgt_height return p.width, p.height @@ -372,7 +408,9 @@ def calculate_base_steps(p, use_denoise_start, use_refiner_start): if len(getattr(p, 'timesteps', [])) > 0: return None cls = shared.sd_model.__class__.__name__ - if 'Flex' in cls or 'Kontext' in cls or 'Edit' in cls: + if 'Flex' in cls or 'Kontext' in cls or 'Edit' in cls or 'Wan' in cls: + steps = p.steps + elif is_modular(): steps = p.steps elif not is_txt2img(): if cls in sd_models.i2i_pipes: @@ -393,7 +431,7 @@ def calculate_base_steps(p, use_denoise_start, use_refiner_start): def calculate_hires_steps(p): cls = shared.sd_model.__class__.__name__ - if 'Flex' in cls or 'HiDreamImageEditingPipeline' in cls or 'Kontext' in cls: + if 'Flex' in cls or 'Kontext' in cls or 'Edit' in cls or 'Wan' in cls: steps = p.steps elif p.hr_second_pass_steps > 0: steps = (p.hr_second_pass_steps // p.denoising_strength) + 1 @@ -407,7 +445,7 @@ def calculate_hires_steps(p): def calculate_refiner_steps(p): cls = shared.sd_model.__class__.__name__ - if 'Flex' in cls or 'HiDreamImageEditingPipeline' in cls or 'Kontext' in cls: + if 'Flex' in cls or 'Kontext' in cls or 'Edit' in cls or 'Wan' in cls: steps = p.steps elif "StableDiffusionXL" in shared.sd_refiner.__class__.__name__: if p.refiner_start > 0 and p.refiner_start < 1: diff --git a/modules/processing_info.py b/modules/processing_info.py index 694fd52b3..e1d4d6379 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -31,13 +31,13 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No if all_subseeds is None: all_subseeds = p.all_subseeds or [p.subseed] while len(all_prompts) <= index: - all_prompts.append(all_prompts[-1]) + all_prompts.insert(0, p.prompt) while len(all_seeds) <= index: - all_seeds.append(all_seeds[-1]) + all_seeds.insert(0, int(p.seed)) while len(all_subseeds) <= index: - all_subseeds.append(all_subseeds[-1]) + all_subseeds.insert(0, int(p.subseed)) while len(all_negative_prompts) <= index: - all_negative_prompts.append(all_negative_prompts[-1]) + all_negative_prompts.insert(0, p.negative_prompt) comment = ', '.join(comments) if comments is not None and type(comments) is list else None ops = list(set(p.ops)) args = { diff --git a/modules/processing_vae.py b/modules/processing_vae.py index aca549a39..933af425f 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -173,7 +173,7 @@ def full_vae_decode(latents, model): log_debug(f'VAE memory: {shared.mem_mon.read()}') vae_name = os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0] if sd_vae.loaded_vae_file is not None else "default" vae_scale_factor = sd_vae.get_vae_scale_factor(model) - shared.log.debug(f'Decode: vae="{vae_name}" scale={vae_scale_factor} upcast={upcast} slicing={getattr(model.vae, "use_slicing", None)} tiling={getattr(model.vae, "use_tiling", None)} latents={list(latents.shape)}:{latents.device}:{latents.dtype} time={t1-t0:.3f}') + shared.log.debug(f'Decode: vae="{vae_name}" scale={vae_scale_factor} upcast={upcast} slicing={getattr(model.vae, "use_slicing", None)} tiling={getattr(model.vae, "use_tiling", None)} latents={list(latents.shape)}:{latents.device} dtype={latents.dtype} time={t1-t0:.3f}') return decoded @@ -221,7 +221,7 @@ def taesd_vae_decode(latents): else: decoded = sd_vae_taesd.decode(latents) t1 = time.time() - shared.log.debug(f'Decode: vae="taesd" latents={latents.shape}:{latents.dtype}:{latents.device} time={t1-t0:.3f}') + shared.log.debug(f'Decode: vae="taesd" latents={latents.shape}:{latents.device} dtype={latents.dtype} time={t1-t0:.3f}') return decoded diff --git a/modules/rocm.py b/modules/rocm.py index 554c3af55..1b03bfd32 100644 --- a/modules/rocm.py +++ b/modules/rocm.py @@ -3,9 +3,9 @@ import sys import ctypes import shutil import subprocess -import importlib.metadata from typing import Union, List from enum import Enum +from functools import wraps def resolve_link(path_: str) -> str: @@ -29,15 +29,30 @@ def load_library_global(path_: str): ctypes.CDLL(path_, mode=ctypes.RTLD_GLOBAL) -def conceal(): - os.environ.pop("ROCM_HOME", None) - os.environ.pop("ROCM_PATH", None) - paths = os.environ["PATH"].split(";") - paths_no_rocm = [] - for path_ in paths: - if "rocm" not in path_.lower(): - paths_no_rocm.append(path_) - os.environ["PATH"] = ";".join(paths_no_rocm) +class Environment: + pass + + +# rocm is installed system-wide +class ROCmEnvironment(Environment): + path: str + + def __init__(self, path: str): + self.path = path + + +# rocm-sdk package is installed +class PythonPackageEnvironment(Environment): + hip: ctypes.CDLL + + def __init__(self): + import _rocm_sdk_core + if sys.platform == "win32": + path = os.path.join(_rocm_sdk_core.__path__[0], "bin", "amdhip64_7.dll") + else: + raise NotImplementedError + # This library will be loaded/used by PyTorch. So it won't make conflicts. + self.hip = ctypes.CDLL(path) class MicroArchitecture(Enum): @@ -80,8 +95,22 @@ class Agent: self.is_apu = (self.gfx_version & 0xFFF0 == 0x1150) or self.gfx_version in (0x801, 0x902, 0x90c, 0x1013, 0x1033, 0x1035, 0x1036, 0x1103,) self.blaslt_supported = os.path.exists(os.path.join(blaslt_tensile_libpath, f"Kernels.so-000-{name}.hsaco" if sys.platform == "win32" else f"extop_{name}.co")) + @property + def therock(self) -> str: + if (self.gfx_version & 0xFFF0) == 0x1100: + return "gfx110X-dgpu" + if self.gfx_version == 0x1151: + return "gfx1151" + if (self.gfx_version & 0xFFF0) == 0x1200: + return "gfx120X-all" + if (self.gfx_version & 0xFFF0) == 0x940: + return "gfx94X-dcgpu" + if self.gfx_version == 0x950: + return "gfx950-dcgpu" + raise RuntimeError(f"Unsupported GPU architecture: {self.name}") + def get_gfx_version(self) -> Union[str, None]: - if self.gfx_version >= 0x1101 and self.gfx_version < 0x1200: + if self.gfx_version >= 0x1100 and self.gfx_version < 0x1200: return "11.0.0" elif self.gfx_version != 0x1030 and self.gfx_version >= 0x1000 and self.gfx_version < 0x1100: # gfx1010 users had to override gfx version to 10.3.0 in Linux @@ -90,26 +119,22 @@ class Agent: return None -def get_version_torch() -> Union[str, None]: - version_ = None - try: - version_ = importlib.metadata.version("torch") - except importlib.metadata.PackageNotFoundError: - return None - if "+rocm" not in version_: # unofficial build, non-rocm torch. - return None - return version_.split("+rocm")[1] +def find() -> Union[Environment, None]: + try: # TheRock + import _rocm_sdk_core # pylint: disable=unused-import + return PythonPackageEnvironment() + except ImportError: + pass + # system-wide installation + hip_path = shutil.which("hipconfig") + if hip_path is not None: + return ROCmEnvironment(dirname(resolve_link(hip_path), 2)) -if sys.platform == "win32": - def find() -> Union[str, None]: - hip_path = shutil.which("hipconfig") - if hip_path is not None: - return dirname(resolve_link(hip_path), 2) - + if sys.platform == "win32": hip_path = os.environ.get("HIP_PATH", None) if hip_path is not None: - return hip_path + return ROCmEnvironment(hip_path) program_files = os.environ.get('ProgramFiles', r'C:\Program Files') hip_path = rf'{program_files}\AMD\ROCm' @@ -146,29 +171,111 @@ if sys.platform == "win32": if latest is None: return None - return os.path.join(hip_path, str(latest)) - - def get_version() -> str: # cannot just run hipconfig as it requires Perl installed on Windows. - return os.path.basename(path) or os.path.basename(os.path.dirname(path)) - - def get_agents() -> List[Agent]: - return [Agent(x.split(' ')[-1].strip()) for x in spawn("hipinfo", cwd=os.path.join(path, 'bin')).split("\n") if x.startswith('gcnArchName:')] - - is_wsl: bool = False - version_torch = None -else: - def find() -> Union[str, None]: - rocm_path = shutil.which("hipconfig") - if rocm_path is not None: - return dirname(resolve_link(rocm_path), 2) + return ROCmEnvironment(os.path.join(hip_path, str(latest))) + else: if not os.path.exists("/opt/rocm"): return None - return resolve_link("/opt/rocm") + return ROCmEnvironment(resolve_link("/opt/rocm")) - def get_version() -> str: - arr = spawn("hipconfig --version", cwd=os.path.join(path, 'bin')).split(".") - return f'{arr[0]}.{arr[1]}' if len(arr) >= 2 else None +def get_version() -> str: + if isinstance(environment, ROCmEnvironment): + # We don't load the hip library that will not be used by PyTorch. + if sys.platform == "win32": + # ROCm is system-wide installed. Assume the version is the folder name. (e.g. C:\Program Files\AMD\ROCm\6.4) + # hipconfig requires Perl + return os.path.basename(environment.path) or os.path.basename(os.path.dirname(environment.path)) + else: + arr = spawn("hipconfig --version", cwd=os.path.join(environment.path, 'bin')).split(".") + return f'{arr[0]}.{arr[1]}' if len(arr) >= 2 else None + else: + # If rocm-sdk package is installed, the hip library may be used by PyTorch. + ver = ctypes.c_int() + environment.hip.hipRuntimeGetVersion(ctypes.byref(ver)) + major = ver.value // 10000000 + minor = (ver.value // 100000) % 100 + #patch = version.value % 100000 + return f"{major}.{minor}" + + +def get_flash_attention_command(agent: Agent) -> str: + default = "git+https://github.com/ROCm/flash-attention" + if agent.gfx_version >= 0x1100 and agent.gfx_version < 0x1200 and os.environ.get("FLASH_ATTENTION_USE_TRITON_ROCM", "false").lower() != "true": + # use the navi_rotary_fix fork because the original doesn't support rotary_emb for transformers + # original: "git+https://github.com/ROCm/flash-attention@howiejay/navi_support" + default = "git+https://github.com/Disty0/flash-attention@navi_rotary_fix" + return "--no-build-isolation " + os.environ.get("FLASH_ATTENTION_PACKAGE", default) + + +if sys.platform == "win32": + def get_agents() -> List[Agent]: + if isinstance(environment, ROCmEnvironment): + out = spawn("amdgpu-arch", cwd=os.path.join(environment.path, 'bin')) + else: + # Assume that amdgpu-arch is in PATH (venv/Scripts/amdgpu-arch.exe) + out = spawn("amdgpu-arch") + out = out.strip() + return [Agent(x.split(' ')[-1].strip()) for x in out.split("\n")] + + def driver_get_agents() -> List[Agent]: + # unsafe and experimental feature + from modules import windows_hip_ffi + hip = windows_hip_ffi.HIP() + count = hip.get_device_count() + agents = [None] * count + for i in range(count): + prop = hip.get_device_properties(i) + name = prop.gcnArchName.decode('utf-8').strip('\x00') + agents[i] = Agent(name) + del hip + return agents + + def postinstall(): + import torch + if torch.version.hip is None: + os.environ.pop("ROCM_HOME", None) + os.environ.pop("ROCM_PATH", None) + paths = os.environ["PATH"].split(";") + paths_no_rocm = [] + for path_ in paths: + if "rocm" not in path_.lower(): + paths_no_rocm.append(path_) + os.environ["PATH"] = ";".join(paths_no_rocm) + return + + def rocm_init(): + try: + import torch + import numpy as np + + original_cholesky_ex = torch.linalg.cholesky_ex + @wraps(original_cholesky_ex) + def cholesky_ex(A: torch.Tensor, upper=False, check_errors=False, out=None) -> torch.return_types.linalg_cholesky_ex: + assert not check_errors + return_device = A.device + L = torch.from_numpy(np.linalg.cholesky(A.to("cpu").numpy(), upper=upper)).to(return_device) + info = torch.tensor(0, dtype=torch.int32, device=return_device) + if out is not None: + out[0].copy_(L) + out[1].copy_(info) + return torch.return_types.linalg_cholesky_ex((L, info), {}) + torch.linalg.cholesky_ex = cholesky_ex + + original_cholesky = torch.linalg.cholesky + @wraps(original_cholesky) + def cholesky(A: torch.Tensor, upper=False, out=None) -> torch.Tensor: + return_device = A.device + L = torch.from_numpy(np.linalg.cholesky(A.to("cpu").numpy(), upper=upper)).to(return_device) + if out is not None: + out.copy_(L) + return L + torch.linalg.cholesky = cholesky + except Exception as e: + return False, e + return True, None + + is_wsl: bool = False +else: def get_agents() -> List[Agent]: try: agents = spawn("rocm_agent_enumerator").split("\n") @@ -178,41 +285,32 @@ else: agents = [x.strip().split(" ")[-1] for x in agents if x.startswith(' Name:') and "CPU" not in x] return [Agent(x) for x in agents] - def load_hsa_runtime() -> None: - try: - # Preload stdc++ library. This will ignore Anaconda stdc++ library. - load_library_global("/lib/x86_64-linux-gnu/libstdc++.so.6") - # Use tcmalloc if possible. - load_library_global("/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4") - except OSError: - pass - # Preload HSA Runtime library. - load_library_global("/opt/rocm/lib/libhsa-runtime64.so") + def postinstall(): + if is_wsl: + try: + if shutil.which("conda") is not None: + # Preload stdc++ library. This will bypass Anaconda stdc++ library. + load_library_global("/lib/x86_64-linux-gnu/libstdc++.so.6") + # Preload rocr4wsl. The user don't have to replace the library file. + load_library_global("/opt/rocm/lib/libhsa-runtime64.so") + except OSError: + pass - def set_blaslt_enabled(enabled: bool) -> None: - if enabled: - os.environ["HIPBLASLT_TENSILE_LIBPATH"] = blaslt_tensile_libpath - else: - os.environ["TORCH_BLAS_PREFER_HIPBLASLT"] = "0" - - def get_blaslt_enabled() -> bool: - return version == version_torch and bool(int(os.environ.get("TORCH_BLAS_PREFER_HIPBLASLT", "1"))) - - def get_flash_attention_command(agent: Agent): - default = "git+https://github.com/ROCm/flash-attention" - if agent.gfx_version >= 0x1100 and agent.gfx_version < 0x1200 and os.environ.get("FLASH_ATTENTION_USE_TRITON_ROCM", "false").lower() != "true": - # use the navi_rotary_fix fork because the original doesn't support rotary_emb for transformers - # original: "git+https://github.com/ROCm/flash-attention@howiejay/navi_support" - default = "git+https://github.com/Disty0/flash-attention@navi_rotary_fix" - return "--no-build-isolation " + os.environ.get("FLASH_ATTENTION_PACKAGE", default) + def rocm_init(): + return True, None is_wsl: bool = os.environ.get('WSL_DISTRO_NAME', 'unknown' if spawn('wslpath -w /') else None) is not None - version_torch = get_version_torch() -path = find() +environment = None blaslt_tensile_libpath = "" is_installed = False version = None -if path is not None: - blaslt_tensile_libpath = os.environ.get("HIPBLASLT_TENSILE_LIBPATH", os.path.join(path, "bin" if sys.platform == "win32" else "lib", "hipblaslt", "library")) - is_installed = True - version = get_version() + +def refresh(): + global environment, blaslt_tensile_libpath, is_installed, version # pylint: disable=global-statement + environment = find() + if environment is not None: + if isinstance(environment, ROCmEnvironment): + blaslt_tensile_libpath = os.environ.get("HIPBLASLT_TENSILE_LIBPATH", os.path.join(environment.path, "bin" if sys.platform == "win32" else "lib", "hipblaslt", "library")) + is_installed = True + version = get_version() +refresh() diff --git a/modules/sd_detect.py b/modules/sd_detect.py index b945a6457..838c68770 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -115,6 +115,8 @@ def guess_by_name(fn, current_guess): return 'Kandinsky 3.0' elif 'hunyuanimage' in fn.lower(): return 'HunyuanImage' + elif 'sdxl-turbo' in fn.lower() or 'stable-diffusion-xl' in fn.lower(): + return 'Stable Diffusion XL' return current_guess diff --git a/modules/sd_hijack_vae.py b/modules/sd_hijack_vae.py index edd8427e6..2563ccfed 100644 --- a/modules/sd_hijack_vae.py +++ b/modules/sd_hijack_vae.py @@ -18,7 +18,7 @@ def hijack_vae_decode(*args, **kwargs): latents = args[0].to(device=devices.device, dtype=shared.sd_model.vae.dtype) # upcast to vae dtype res = shared.sd_model.vae.orig_decode(latents, *args[1:], **kwargs) t1 = time.time() - shared.log.debug(f'Decode: vae={shared.sd_model.vae.__class__.__name__} slicing={getattr(shared.sd_model.vae, "use_slicing", None)} tiling={getattr(shared.sd_model.vae, "use_tiling", None)} latents={list(latents.shape)}:{latents.device}:{latents.dtype} time={t1-t0:.3f}') + shared.log.debug(f'Decode: vae={shared.sd_model.vae.__class__.__name__} slicing={getattr(shared.sd_model.vae, "use_slicing", None)} tiling={getattr(shared.sd_model.vae, "use_tiling", None)} latents={list(latents.shape)}:{latents.device} dtype={latents.dtype} time={t1-t0:.3f}') else: res = shared.sd_model.vae.orig_decode(*args, **kwargs) except Exception as e: diff --git a/modules/sd_models.py b/modules/sd_models.py index a251495c3..e759ac56f 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -411,65 +411,70 @@ def load_diffuser_folder(model_type, pipeline, checkpoint_info, diffusers_load_c files = shared.walk_files(checkpoint_info.path, ['.safetensors', '.bin', '.ckpt']) if 'variant' not in diffusers_load_config and any('diffusion_pytorch_model.fp16' in f for f in files): # deal with diffusers lack of variant fallback when loading diffusers_load_config['variant'] = 'fp16' - if (model_type is not None) and (pipeline is not None) and ('ONNX' in model_type): # forced pipeline - try: - sd_model = pipeline.from_pretrained(checkpoint_info.path) - except Exception as e: - shared.log.error(f'Load {op}: type=ONNX path="{checkpoint_info.path}" {e}') - if debug_load: - errors.display(e, 'Load') - return None - else: - err1, err2, err3 = None, None, None - if os.path.exists(checkpoint_info.path) and os.path.isdir(checkpoint_info.path): - if os.path.exists(os.path.join(checkpoint_info.path, 'unet', 'diffusion_pytorch_model.bin')): - shared.log.debug(f'Load {op}: type=pickle') - diffusers_load_config['use_safetensors'] = False + + err0, err1, err2, err3 = None, None, None, None + if os.path.exists(checkpoint_info.path) and os.path.isdir(checkpoint_info.path): + if os.path.exists(os.path.join(checkpoint_info.path, 'unet', 'diffusion_pytorch_model.bin')): + shared.log.debug(f'Load {op}: type=pickle') + diffusers_load_config['use_safetensors'] = False + if debug_load: + shared.log.debug(f'Load {op}: args={diffusers_load_config}') + + try: #0 - using detected model type and pipeline + if (model_type is not None) and (pipeline is not None): + sd_model = pipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model.model_type = sd_model.__class__.__name__ + except Exception as e: + err0 = e if debug_load: - shared.log.debug(f'Load {op}: args={diffusers_load_config}') - try: # 1 - autopipeline, best choice but not all pipelines are available - try: + errors.display(e, 'Load Detected') + + try: # 1 - autopipeline, best choice but not all pipelines are available + try: + if err0 is not None: sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) sd_model.model_type = sd_model.__class__.__name__ - except ValueError as e: - if 'no variant default' in str(e): - shared.log.warning(f'Load {op}: variant={diffusers_load_config["variant"]} model="{checkpoint_info.path}" using default variant') - diffusers_load_config.pop('variant', None) - sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) - sd_model.model_type = sd_model.__class__.__name__ - elif 'safetensors found in directory' in str(err1): - shared.log.warning(f'Load {op}: type=pickle') - diffusers_load_config['use_safetensors'] = False - sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) - sd_model.model_type = sd_model.__class__.__name__ - else: - raise ValueError from e # reraise - except Exception as e: - err1 = e - if debug_load: - errors.display(e, 'Load AutoPipeline') - # shared.log.error(f'AutoPipeline: {e}') - try: # 2 - diffusion pipeline, works for most non-linked pipelines - if err1 is not None: - sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + except ValueError as e: + if 'no variant default' in str(e): + shared.log.warning(f'Load {op}: variant={diffusers_load_config["variant"]} model="{checkpoint_info.path}" using default variant') + diffusers_load_config.pop('variant', None) + sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) sd_model.model_type = sd_model.__class__.__name__ - except Exception as e: - err2 = e - if debug_load: - errors.display(e, "Load DiffusionPipeline") - # shared.log.error(f'DiffusionPipeline: {e}') - try: # 3 - try basic pipeline just in case - if err2 is not None: - sd_model = diffusers.StableDiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + elif 'safetensors found in directory' in str(err1): + shared.log.warning(f'Load {op}: type=pickle') + diffusers_load_config['use_safetensors'] = False + sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) sd_model.model_type = sd_model.__class__.__name__ - except Exception as e: - err3 = e # ignore last error - shared.log.error(f"StableDiffusionPipeline: {e}") - if debug_load: - errors.display(e, "Load StableDiffusionPipeline") - if err3 is not None: - shared.log.error(f'Load {op}: {checkpoint_info.path} auto={err1} diffusion={err2}') - return None + else: + raise ValueError from e # reraise + except Exception as e: + err1 = e + if debug_load: + errors.display(e, 'Load AutoPipeline') + + try: # 2 - diffusion pipeline, works for most non-linked pipelines + if err1 is not None: + sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model.model_type = sd_model.__class__.__name__ + except Exception as e: + err2 = e + if debug_load: + errors.display(e, "Load DiffusionPipeline") + + try: # 3 - try basic pipeline just in case + if err2 is not None: + sd_model = diffusers.StableDiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model.model_type = sd_model.__class__.__name__ + except Exception as e: + err3 = e # ignore last error + shared.log.error(f"StableDiffusionPipeline: {e}") + if debug_load: + errors.display(e, "Load StableDiffusionPipeline") + + if err3 is not None: + shared.log.error(f'Load {op}: {checkpoint_info.path} detected={err0} auto={err1} diffusion={err2} base={err3}') + return None + return sd_model @@ -667,7 +672,7 @@ def load_diffuser(checkpoint_info=None, op='model', revision=None): # pylint: di sd_model.scheduler.name = 'DDIM' if hasattr(sd_model, "unet") and model_type not in ['Stable Cascade']: # others calls load_diffuser again - sd_unet.load_unet(sd_model) + sd_unet.load_unet(sd_model, checkpoint_info.path) add_noise_pred_to_diffusers_callback(sd_model) @@ -738,6 +743,7 @@ class DiffusersTaskType(Enum): IMAGE_2_IMAGE = 2 INPAINTING = 3 INSTRUCT = 4 + MODULAR = 5 def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType: @@ -748,6 +754,8 @@ def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType: return DiffusersTaskType.IMAGE_2_IMAGE elif 'Instruct' in cls: return DiffusersTaskType.INSTRUCT + elif 'Modular' in cls: + return DiffusersTaskType.MODULAR elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING.values(): return DiffusersTaskType.IMAGE_2_IMAGE elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING.values(): @@ -945,6 +953,9 @@ def set_diffuser_pipe(pipe, new_pipe_type): if get_diffusers_task(pipe) == new_pipe_type: return pipe + if get_diffusers_task(pipe) == DiffusersTaskType.MODULAR: + return pipe + # skip specific pipelines cls = pipe.__class__.__name__ if cls in pipe_switch_task_exclude: @@ -1029,7 +1040,13 @@ def set_diffusers_attention(pipe, quiet:bool=False): return # other models uses their own attention processor if pipe.__class__.__name__.startswith("StableDiffusion") and hasattr(pipe, "unet"): - pipe.unet.set_attn_processor(attention) + try: + pipe.unet.set_attn_processor(attention) + except Exception as e: + if 'Nunchaku' in pipe.unet.__class__.__name__: + pass + else: + shared.log.error(f"Attention: {name if name is not None else attention.__class__.__name__} pipe={pipe.__class__.__name__} {e}") elif not quiet: shared.log.warning(f"Attention: {name if name is not None else attention.__class__.__name__} is not compatible with {pipe.__class__.__name__}") @@ -1053,17 +1070,19 @@ def set_diffusers_attention(pipe, quiet:bool=False): pipe.enable_xformers_memory_efficient_attention() else: shared.log.warning(f"Attention: xFormers is not compatible with {pipe.__class__.__name__}") - elif shared.opts.cross_attention_optimization == "Split attention": - if hasattr(pipe, "enable_attention_slicing"): - pipe.enable_attention_slicing() - else: - shared.log.warning(f"Attention: Split attention is not compatible with {pipe.__class__.__name__}") elif shared.opts.cross_attention_optimization == "Batch matrix-matrix": set_attn(pipe, p.AttnProcessor(), name="Batch matrix-matrix") elif shared.opts.cross_attention_optimization == "Dynamic Attention BMM": from modules.sd_hijack_dynamic_atten import DynamicAttnProcessorBMM set_attn(pipe, DynamicAttnProcessorBMM(), name="Dynamic Attention BMM") + if shared.opts.attention_slicing != "Default" and hasattr(pipe, "enable_attention_slicing") and hasattr(pipe, "disable_attention_slicing"): + if shared.opts.attention_slicing: + pipe.enable_attention_slicing() + else: + pipe.disable_attention_slicing() + shared.log.debug(f"Attention: slicing={shared.opts.attention_slicing}") + pipe.current_attn_name = shared.opts.cross_attention_optimization diff --git a/modules/sd_offload.py b/modules/sd_offload.py index bf951e2c5..19e59de3d 100644 --- a/modules/sd_offload.py +++ b/modules/sd_offload.py @@ -18,6 +18,7 @@ offload_post = ['h1'] offload_hook_instance = None balanced_offload_exclude = ['CogView4Pipeline', 'MeissonicPipeline'] accelerate_dtype_byte_size = None +move_stream = None def dtype_byte_size(dtype: torch.dtype): @@ -176,7 +177,7 @@ class OffloadHook(accelerate.hooks.ModelHook): self.last_post = None self.last_cls = None gpu = f'{(shared.gpu_memory * shared.opts.diffusers_offload_min_gpu_memory):.2f}-{(shared.gpu_memory * shared.opts.diffusers_offload_max_gpu_memory):.2f}:{shared.gpu_memory:.2f}' - shared.log.info(f'Offload: type=balanced op=init watermark={self.min_watermark}-{self.max_watermark} gpu={gpu} cpu={shared.cpu_memory:.3f} limit={shared.opts.cuda_mem_fraction:.2f} always={self.offload_always} never={self.offload_never} pre={shared.opts.diffusers_offload_pre}') + shared.log.info(f'Offload: type=balanced op=init watermark={self.min_watermark}-{self.max_watermark} gpu={gpu} cpu={shared.cpu_memory:.3f} limit={shared.opts.cuda_mem_fraction:.2f} always={self.offload_always} never={self.offload_never} pre={shared.opts.diffusers_offload_pre} streams={shared.opts.diffusers_offload_streams}') self.validate() super().__init__() @@ -206,12 +207,18 @@ class OffloadHook(accelerate.hooks.ModelHook): return False if hasattr(module, 'nets') and any(hasattr(n, "offload_never") for n in module.nets): return False + if shared.sd_model_type.lower() in [m.lower().strip() for m in re.split(r'[ ,]+', shared.opts.models_not_to_offload)]: + return False return True def pre_forward(self, module, *args, **kwargs): _id = id(module) - if (self.last_pre != _id) and (module.__class__.__name__ != self.last_cls) and self.offload_allowed(module): # offload every other module first time when new module starts pre-forward + + do_offload = (self.last_pre != _id) or (module.__class__.__name__ != self.last_cls) + + if do_offload and self.offload_allowed(module): # offload every other module first time when new module starts pre-forward if shared.opts.diffusers_offload_pre: + t0 = time.time() debug_move(f'Offload: type=balanced op=pre module={module.__class__.__name__}') for pipe in get_pipe_variants(): for module_name in get_module_names(pipe): @@ -220,15 +227,16 @@ class OffloadHook(accelerate.hooks.ModelHook): if (_id != id(module_instance)) and (module_cls not in self.offload_never) and (not devices.same_device(module_instance.device, devices.cpu)): apply_balanced_offload_to_module(module_instance, op='pre') self.last_cls = module.__class__.__name__ - self.last_pre = _id + process_timer.add('offload', time.time() - t0) if not devices.same_device(module.device, devices.device): # move-to-device + t0 = time.time() device_index = torch.device(devices.device).index if device_index is None: device_index = 0 max_memory = { device_index: self.gpu, "cpu": self.cpu } device_map = getattr(module, "balanced_offload_device_map", None) - if device_map is None or max_memory != getattr(module, "balanced_offload_max_memory", None): + if (device_map is None) or (max_memory != getattr(module, "balanced_offload_max_memory", None)): device_map = accelerate.infer_auto_device_map(module, max_memory=max_memory) offload_dir = getattr(module, "offload_dir", os.path.join(shared.opts.accelerate_offload_path, module.__class__.__name__)) if devices.backend == "directml": @@ -241,13 +249,15 @@ class OffloadHook(accelerate.hooks.ModelHook): module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access module.balanced_offload_device_map = device_map module.balanced_offload_max_memory = max_memory + process_timer.add('onload', time.time() - t0) if debug: - for pipe in get_pipe_variants(): + for _i, pipe in enumerate(get_pipe_variants()): for module_name in get_module_names(pipe): module_instance = getattr(pipe, module_name, None) - shared.log.trace(f'Offload: type=balanced op=pre:status module={module_instance.__class__.__name__} device={module_instance.device} dtype={module_instance.dtype}') + shared.log.trace(f'Offload: type=balanced op=pre:status forward={module.__class__.__name__} module={module_name} class={module_instance.__class__.__name__} pipe={_i} device={module_instance.device} dtype={module_instance.dtype}') + self.last_pre = _id return args, kwargs def post_forward(self, module, output): @@ -283,6 +293,7 @@ def get_module_names(pipe=None, exclude=[]): modules_names = get_signature(pipe).keys() modules_names = [m for m in modules_names if m not in exclude and not m.startswith('_')] modules_names = [m for m in modules_names if isinstance(getattr(pipe, m, None), torch.nn.Module)] + modules_names = sorted(set(modules_names)) return modules_names @@ -308,6 +319,17 @@ def get_module_sizes(pipe=None, exclude=[]): def move_module_to_cpu(module, op='unk', force:bool=False): + def do_move(module): + if shared.opts.diffusers_offload_streams: + global move_stream # pylint: disable=global-statement + if move_stream is None: + move_stream = torch.cuda.Stream(device=devices.device) + with torch.cuda.stream(move_stream): + module = module.to(devices.cpu) + else: + module = module.to(devices.cpu) + return module + try: module_name = getattr(module, "module_name", module.__class__.__name__) module_size = offload_hook_instance.offload_map.get(module_name, offload_hook_instance.model_size()) @@ -318,17 +340,17 @@ def move_module_to_cpu(module, op='unk', force:bool=False): op = f'{op}:skip' if force: op = f'{op}:force' - module = module.to(devices.cpu) + module = do_move(module) used_gpu -= module_size elif module_cls in offload_hook_instance.offload_never: op = f'{op}:never' elif module_cls in offload_hook_instance.offload_always: op = f'{op}:always' - module = module.to(devices.cpu) + module = do_move(module) used_gpu -= module_size elif perc_gpu > shared.opts.diffusers_offload_min_gpu_memory: op = f'{op}:mem' - module = module.to(devices.cpu) + module = do_move(module) used_gpu -= module_size if debug: quant = getattr(module, "quantization_method", None) diff --git a/modules/sd_unet.py b/modules/sd_unet.py index 2bd1741f4..6d3d6c936 100644 --- a/modules/sd_unet.py +++ b/modules/sd_unet.py @@ -1,5 +1,5 @@ import os -from modules import shared, devices, files_cache, sd_models +from modules import shared, devices, files_cache, sd_models, model_quant unet_dict = {} @@ -8,22 +8,55 @@ failed_unet = [] debug = os.environ.get('SD_LOAD_DEBUG', None) is not None -dit_models = ['Flux', 'StableDiffusion3', 'HiDream', 'Lumina2', 'Chroma', 'Wan'] +dit_models = ['Flux', 'StableDiffusion3', 'HiDream', 'Lumina2', 'Chroma', 'Wan', 'Qwen'] -def load_unet(model): +def load_unet_sdxl_nunchaku(repo_id): + try: + from nunchaku.models.unets.unet_sdxl import NunchakuSDXLUNet2DConditionModel + except Exception: + shared.log.error(f'Load module: quant=Nunchaku module=unet repo="{repo_id}" low nunchaku version') + return None + if 'turbo' in repo_id.lower(): + nunchaku_repo = 'nunchaku-tech/nunchaku-sdxl-turbo/svdq-int4_r32-sdxl-turbo.safetensors' + else: + nunchaku_repo = 'nunchaku-tech/nunchaku-sdxl/svdq-int4_r32-sdxl.safetensors' + + shared.log.debug(f'Load module: quant=Nunchaku module=unet repo="{nunchaku_repo}" offload={shared.opts.nunchaku_offload}') + unet = NunchakuSDXLUNet2DConditionModel.from_pretrained( + nunchaku_repo, + offload=shared.opts.nunchaku_offload, + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + ) + unet.quantization_method = 'SVDQuant' + return unet + + +def load_unet(model, repo_id:str=None): global loaded_unet # pylint: disable=global-statement + + if ("StableDiffusionXLPipeline" in model.__class__.__name__) and (('stable-diffusion-xl-base' in repo_id) or ('sdxl-turbo' in repo_id)): + if model_quant.check_nunchaku('Model'): + unet = load_unet_sdxl_nunchaku(repo_id) + if unet is not None: + model.unet = unet + return + if shared.opts.sd_unet == 'Default' or shared.opts.sd_unet == 'None': return + if shared.opts.sd_unet not in list(unet_dict): shared.log.error(f'Load module: type=UNet not found: {shared.opts.sd_unet}') return + config_file = os.path.splitext(unet_dict[shared.opts.sd_unet])[0] + '.json' if os.path.exists(config_file): config = shared.readfile(config_file) else: config = None config_file = 'default' + try: if shared.opts.sd_unet == loaded_unet or shared.opts.sd_unet in failed_unet: pass diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index a7829d44d..450b43d43 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -75,6 +75,10 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz use_quantized_matmul = group_channel_size >= 32 and output_channel_size >= 32 if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: use_quantized_matmul = output_channel_size % 16 == 0 and group_channel_size % 16 == 0 + if use_quantized_matmul and dtype_dict[weights_dtype]["num_bits"] == 8: + result_shape = layer.weight.shape + layer.weight.data = layer.weight.flatten(1,-1) + reduction_axes = -1 elif layer_class_name in conv_transpose_types: if not quant_conv: return layer @@ -128,7 +132,8 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz num_of_groups = int(num_of_groups) if num_of_groups > 1: - result_shape = layer.weight.shape + if result_shape is None: + result_shape = layer.weight.shape new_shape = list(result_shape) if is_conv_type: # output_channel_size, channel_size, X, X @@ -165,17 +170,15 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz re_quantize_for_matmul = (num_of_groups > 1 or zero_point is not None) if use_quantized_matmul and not re_quantize_for_matmul: - if is_conv_type: - result_shape = layer.weight.shape - layer.weight.data = layer.weight.reshape(output_channel_size, -1) scale.transpose_(0,1) layer.weight.transpose_(0,1) - if not dtype_dict[weights_dtype]["is_integer"]: - weight_stride = layer.weight.stride() - if not (weight_stride[0] == 1 and weight_stride[1] > 1): - layer.weight.data = layer.weight.t().contiguous().t() - if not use_tensorwise_fp8_matmul: - scale = scale.to(torch.float32) + if layer.weight.is_contiguous(): + if devices.backend != "ipex": + layer.weight.data = layer.weight.t_().contiguous().t_() + elif devices.backend == "ipex": + layer.weight.data = layer.weight.contiguous() + if not use_tensorwise_fp8_matmul and not dtype_dict[weights_dtype]["is_integer"]: + scale = scale.to(torch.float32) layer.sdnq_dequantizer = dequantizer_dict[weights_dtype]( scale=scale, @@ -402,7 +405,7 @@ class SDNQQuantizer(DiffusersQuantizer): def _process_model_after_weight_loading(self, model, **kwargs): # pylint: disable=unused-argument if shared.opts.diffusers_offload_mode != "none": model = model.to(devices.cpu) - devices.torch_gc(force=True, reason='sdnq') + devices.torch_gc(force=True, reason="sdnq") return model def get_accelerator_warm_up_factor(self): @@ -420,6 +423,12 @@ class SDNQQuantizer(DiffusersQuantizer): """ return config + def update_ep_plan(self, config): + """ + needed for transformers compatibilty, no-op function + """ + return config + def update_unexpected_keys(self, model, unexpected_keys: List[str], prefix: str) -> List[str]: # pylint: disable=unused-argument """ needed for transformers compatibilty, no-op function @@ -432,6 +441,12 @@ class SDNQQuantizer(DiffusersQuantizer): """ return missing_keys + def update_state_dict_with_metadata(self, state_dict: dict, metadata: dict) -> dict: # pylint: disable=unused-argument + """ + needed for transformers compatibilty, no-op function + """ + return state_dict + def update_expected_keys(self, model, expected_keys: List[str], loaded_keys: List[str]) -> List[str]: # pylint: disable=unused-argument """ needed for transformers compatibilty, no-op function diff --git a/modules/sdnq/common.py b/modules/sdnq/common.py index fab48c4b9..783a7ab2f 100644 --- a/modules/sdnq/common.py +++ b/modules/sdnq/common.py @@ -1,7 +1,9 @@ # pylint: disable=redefined-builtin,no-member,protected-access import os +from functools import partial import torch + from modules import shared torch_version = float(torch.__version__[:3]) @@ -32,7 +34,7 @@ if hasattr(torch, "float8_e5m2fnuz"): dtype_dict["float8_e5m2fnuz"] = {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": "fp8", "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False} use_torch_compile = shared.opts.sdnq_dequantize_compile # this setting requires a full restart of the webui to apply -use_tensorwise_fp8_matmul = os.environ.get('SDNQ_USE_TENSORWISE_FP8_MATMUL', "1").lower() not in {"0", "false", "no"} # row-wise FP8 only exist on H100 hardware, sdnq will use software row-wise with tensorwise hardware with this setting +use_tensorwise_fp8_matmul = os.environ.get("SDNQ_USE_TENSORWISE_FP8_MATMUL", "1").lower() not in {"0", "false", "no"} # row-wise FP8 only exist on H100 hardware, sdnq will use software row-wise with tensorwise hardware with this setting linear_types = ("Linear",) conv_types = ("Conv1d", "Conv2d", "Conv3d") @@ -42,3 +44,7 @@ allowed_types = linear_types + conv_types + conv_transpose_types if use_torch_compile: torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) torch._dynamo.config.accumulated_recompile_limit = max(8192, torch._dynamo.config.accumulated_recompile_limit) + compile_func = partial(torch.compile, fullgraph=True) +else: + def compile_func(fn, **kwargs): # pylint: disable=unused-argument + return fn diff --git a/modules/sdnq/dequantizer.py b/modules/sdnq/dequantizer.py index 9ac902c57..be68ab96d 100644 --- a/modules/sdnq/dequantizer.py +++ b/modules/sdnq/dequantizer.py @@ -4,7 +4,7 @@ from typing import Tuple import torch -from .common import dtype_dict, use_torch_compile +from .common import dtype_dict, compile_func from .packed_int import pack_int_symetric, unpack_int_symetric, pack_int_asymetric, unpack_int_asymetric @@ -226,21 +226,11 @@ dequantizer_dict = { } -if use_torch_compile: - dequantize_asymmetric_compiled = torch.compile(dequantize_asymmetric, fullgraph=True, dynamic=False) - dequantize_symmetric_compiled = torch.compile(dequantize_symmetric, fullgraph=True, dynamic=False) - dequantize_packed_int_asymmetric_compiled = torch.compile(dequantize_packed_int_asymmetric, fullgraph=True, dynamic=False) - dequantize_packed_int_symmetric_compiled = torch.compile(dequantize_packed_int_symmetric, fullgraph=True, dynamic=False) - re_quantize_matmul_asymmetric_compiled = torch.compile(re_quantize_matmul_asymmetric, fullgraph=True, dynamic=False) - re_quantize_matmul_symmetric_compiled = torch.compile(re_quantize_matmul_symmetric, fullgraph=True, dynamic=False) - re_quantize_matmul_packed_int_asymmetric_compiled = torch.compile(re_quantize_matmul_packed_int_asymmetric, fullgraph=True, dynamic=False) - re_quantize_matmul_packed_int_symmetric_compiled = torch.compile(re_quantize_matmul_packed_int_symmetric, fullgraph=True, dynamic=False) -else: - dequantize_asymmetric_compiled = dequantize_asymmetric - dequantize_symmetric_compiled = dequantize_symmetric - dequantize_packed_int_asymmetric_compiled = dequantize_packed_int_asymmetric - dequantize_packed_int_symmetric_compiled = dequantize_packed_int_symmetric - re_quantize_matmul_asymmetric_compiled = re_quantize_matmul_asymmetric - re_quantize_matmul_symmetric_compiled = re_quantize_matmul_symmetric - re_quantize_matmul_packed_int_asymmetric_compiled = re_quantize_matmul_packed_int_asymmetric - re_quantize_matmul_packed_int_symmetric_compiled = re_quantize_matmul_packed_int_symmetric +dequantize_asymmetric_compiled = compile_func(dequantize_asymmetric) +dequantize_symmetric_compiled = compile_func(dequantize_symmetric) +dequantize_packed_int_asymmetric_compiled = compile_func(dequantize_packed_int_asymmetric) +dequantize_packed_int_symmetric_compiled = compile_func(dequantize_packed_int_symmetric) +re_quantize_matmul_asymmetric_compiled = compile_func(re_quantize_matmul_asymmetric) +re_quantize_matmul_symmetric_compiled = compile_func(re_quantize_matmul_symmetric) +re_quantize_matmul_packed_int_asymmetric_compiled = compile_func(re_quantize_matmul_packed_int_asymmetric) +re_quantize_matmul_packed_int_symmetric_compiled = compile_func(re_quantize_matmul_packed_int_symmetric) diff --git a/modules/sdnq/layers/conv/conv_fp8.py b/modules/sdnq/layers/conv/conv_fp8.py index cfce8994a..7a83bb3a8 100644 --- a/modules/sdnq/layers/conv/conv_fp8.py +++ b/modules/sdnq/layers/conv/conv_fp8.py @@ -4,8 +4,9 @@ from typing import List import torch -from ...common import use_torch_compile # noqa: TID252 +from ...common import compile_func # noqa: TID252 from ..linear.linear_fp8 import quantize_fp8_matmul_input # noqa: TID252 +from ..linear.forward import check_mats # noqa: TID252 from .forward import get_conv_args, process_conv_input @@ -23,6 +24,7 @@ def conv_fp8_matmul( return_dtype = input.dtype input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) input, input_scale = quantize_fp8_matmul_input(input) + input, weight = check_mats(input, weight) if groups == 1: if bias is not None and bias.dtype != torch.bfloat16: @@ -68,5 +70,4 @@ def quantized_conv_forward_fp8_matmul(self, input) -> torch.FloatTensor: ) -if use_torch_compile: - conv_fp8_matmul = torch.compile(conv_fp8_matmul, fullgraph=True, dynamic=False) +conv_fp8_matmul = compile_func(conv_fp8_matmul) diff --git a/modules/sdnq/layers/conv/conv_fp8_tensorwise.py b/modules/sdnq/layers/conv/conv_fp8_tensorwise.py index a21699855..e3b005622 100644 --- a/modules/sdnq/layers/conv/conv_fp8_tensorwise.py +++ b/modules/sdnq/layers/conv/conv_fp8_tensorwise.py @@ -4,9 +4,10 @@ from typing import List import torch -from ...common import use_torch_compile # noqa: TID252 +from ...common import compile_func # noqa: TID252 from ...dequantizer import dequantize_symmetric, dequantize_symmetric_with_bias # noqa: TID252 from ..linear.linear_fp8_tensorwise import quantize_fp8_matmul_input_tensorwise # noqa: TID252 +from ..linear.forward import check_mats # noqa: TID252 from .forward import get_conv_args, process_conv_input @@ -24,6 +25,7 @@ def conv_fp8_matmul_tensorwise( return_dtype = input.dtype input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) + input, weight = check_mats(input, weight) dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) if groups == 1: @@ -63,5 +65,4 @@ def quantized_conv_forward_fp8_matmul_tensorwise(self, input) -> torch.FloatTens ) -if use_torch_compile: - conv_fp8_matmul_tensorwise = torch.compile(conv_fp8_matmul_tensorwise, fullgraph=True, dynamic=False) +conv_fp8_matmul_tensorwise = compile_func(conv_fp8_matmul_tensorwise) diff --git a/modules/sdnq/layers/conv/conv_int8.py b/modules/sdnq/layers/conv/conv_int8.py index cd18b1db6..260acd170 100644 --- a/modules/sdnq/layers/conv/conv_int8.py +++ b/modules/sdnq/layers/conv/conv_int8.py @@ -4,10 +4,11 @@ from typing import List import torch -from ...common import use_torch_compile # noqa: TID252 +from ...common import compile_func # noqa: TID252 from ...packed_int import unpack_int_symetric # noqa: TID252 from ...dequantizer import dequantize_symmetric, dequantize_symmetric_with_bias # noqa: TID252 from ..linear.linear_int8 import quantize_int8_matmul_input # noqa: TID252 +from ..linear.forward import check_mats # noqa: TID252 from .forward import get_conv_args, process_conv_input @@ -29,6 +30,7 @@ def conv_int8_matmul( input, scale = quantize_int8_matmul_input(input, scale) if quantized_weight_shape is not None: weight = unpack_int_symetric(weight, quantized_weight_shape, weights_dtype, dtype=torch.int8) + input, weight = check_mats(input, weight) if groups == 1: result = torch._int_mm(input, weight) @@ -75,5 +77,4 @@ def quantized_conv_forward_int8_matmul(self, input) -> torch.FloatTensor: ) -if use_torch_compile: - conv_int8_matmul = torch.compile(conv_int8_matmul, fullgraph=True, dynamic=False) +conv_int8_matmul = compile_func(conv_int8_matmul) diff --git a/modules/sdnq/layers/linear/forward.py b/modules/sdnq/layers/linear/forward.py index 20224204c..18009c27b 100644 --- a/modules/sdnq/layers/linear/forward.py +++ b/modules/sdnq/layers/linear/forward.py @@ -1,7 +1,19 @@ # pylint: disable=relative-beyond-top-level,redefined-builtin,protected-access +from typing import Tuple + import torch +def check_mats(input: torch.Tensor, weight: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + input = input.contiguous() + if weight.is_contiguous(): + if weight.device.type != "xpu": + weight = weight.t().contiguous().t() + elif weight.device.type == "xpu": + weight = weight.contiguous() + return input, weight + + def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor: return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight), self.bias) diff --git a/modules/sdnq/layers/linear/linear_fp8.py b/modules/sdnq/layers/linear/linear_fp8.py index 5707ad225..d1845b1e7 100644 --- a/modules/sdnq/layers/linear/linear_fp8.py +++ b/modules/sdnq/layers/linear/linear_fp8.py @@ -4,8 +4,9 @@ from typing import Tuple import torch -from ...common import use_torch_compile # noqa: TID252 +from ...common import compile_func # noqa: TID252 from ...dequantizer import quantize_fp8 # noqa: TID252 +from .forward import check_mats def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.Tensor, torch.FloatTensor]: @@ -23,6 +24,7 @@ def fp8_matmul( return_dtype = input.dtype output_shape = (*input.shape[:-1], weight.shape[-1]) input, input_scale = quantize_fp8_matmul_input(input) + input, weight = check_mats(input, weight) if bias is not None and bias.dtype != torch.bfloat16: bias = bias.to(dtype=torch.bfloat16) return torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=torch.bfloat16).view(output_shape).to(return_dtype) @@ -34,5 +36,4 @@ def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch return fp8_matmul(input, self.weight, self.bias, self.sdnq_dequantizer.scale) -if use_torch_compile: - fp8_matmul = torch.compile(fp8_matmul, fullgraph=True, dynamic=False) +fp8_matmul = compile_func(fp8_matmul) diff --git a/modules/sdnq/layers/linear/linear_fp8_tensorwise.py b/modules/sdnq/layers/linear/linear_fp8_tensorwise.py index aa80251ae..a35eb0721 100644 --- a/modules/sdnq/layers/linear/linear_fp8_tensorwise.py +++ b/modules/sdnq/layers/linear/linear_fp8_tensorwise.py @@ -4,8 +4,9 @@ from typing import Tuple import torch -from ...common import use_torch_compile # noqa: TID252 +from ...common import compile_func # noqa: TID252 from ...dequantizer import quantize_fp8, dequantize_symmetric, dequantize_symmetric_with_bias # noqa: TID252 +from .forward import check_mats def quantize_fp8_matmul_input_tensorwise(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.Tensor, torch.FloatTensor]: @@ -27,6 +28,7 @@ def fp8_matmul_tensorwise( output_shape = (*input.shape[:-1], weight.shape[-1]) dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) + input, weight = check_mats(input, weight) if bias is not None: return dequantize_symmetric_with_bias(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, bias, return_dtype, output_shape) else: @@ -39,5 +41,4 @@ def quantized_linear_forward_fp8_matmul_tensorwise(self, input: torch.FloatTenso return fp8_matmul_tensorwise(input, self.weight, self.bias, self.sdnq_dequantizer.scale) -if use_torch_compile: - fp8_matmul_tensorwise = torch.compile(fp8_matmul_tensorwise, fullgraph=True, dynamic=False) +fp8_matmul_tensorwise = compile_func(fp8_matmul_tensorwise) diff --git a/modules/sdnq/layers/linear/linear_int8.py b/modules/sdnq/layers/linear/linear_int8.py index 3a1b82c1f..356179b97 100644 --- a/modules/sdnq/layers/linear/linear_int8.py +++ b/modules/sdnq/layers/linear/linear_int8.py @@ -4,9 +4,10 @@ from typing import Tuple import torch -from ...common import use_torch_compile # noqa: TID252 +from ...common import compile_func # noqa: TID252 from ...packed_int import unpack_int_symetric # noqa: TID252 from ...dequantizer import quantize_int8, dequantize_symmetric, dequantize_symmetric_with_bias # noqa: TID252 +from .forward import check_mats def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.CharTensor, torch.FloatTensor]: @@ -31,6 +32,7 @@ def int8_matmul( return_dtype = input.dtype output_shape = (*input.shape[:-1], weight.shape[-1]) input, scale = quantize_int8_matmul_input(input, scale) + input, weight = check_mats(input, weight) if bias is not None: return dequantize_symmetric_with_bias(torch._int_mm(input, weight), scale, bias, return_dtype, output_shape) else: @@ -50,5 +52,4 @@ def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torc return int8_matmul(input, weight, self.bias, scale, quantized_weight_shape, self.sdnq_dequantizer.weights_dtype) -if use_torch_compile: - int8_matmul = torch.compile(int8_matmul, fullgraph=True, dynamic=False) +int8_matmul = compile_func(int8_matmul) diff --git a/modules/shared.py b/modules/shared.py index a969c81b0..fab234f94 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -70,9 +70,8 @@ state = shared_state.State() # early select backend backend = Backend.DIFFUSERS -if not hasattr(cmd_opts, "use_openvino"): - cmd_opts.use_openvino = False if cmd_opts.use_openvino: # override for openvino + os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX') from modules.intel.openvino import get_device_list as get_openvino_device_list # pylint: disable=ungrouped-imports elif cmd_opts.use_ipex or devices.has_xpu(): from modules.intel.ipex import ipex_init @@ -84,6 +83,11 @@ elif cmd_opts.use_directml: ok, e = directml_init() if not ok: log.error(f'DirectML initialization failed: {e}') +elif cmd_opts.use_rocm or devices.has_rocm(): + from modules.rocm import rocm_init + ok, e = rocm_init() + if not ok: + log.error(f'ROCm initialization failed: {e}') devices.backend = devices.get_backend(cmd_opts) devices.device = devices.get_optimal_device() mem_stat = memory_stats() @@ -155,6 +159,8 @@ options_templates.update(options_section(('sd', "Model Loading"), { })) options_templates.update(options_section(('model_options', "Model Options"), { + "model_modular_sep": OptionInfo("

Modular Pipelines

", "", gr.HTML), + "model_modular_enable": OptionInfo(False, "Enable modular pipelines (experimental)"), "model_sd3_sep": OptionInfo("

Stable Diffusion 3.x

", "", gr.HTML), "model_sd3_disable_te5": OptionInfo(False, "Disable T5 text encoder"), "model_h1_sep": OptionInfo("

HiDream

", "", gr.HTML), @@ -170,9 +176,11 @@ options_templates.update(options_section(('offload', "Model Offloading"), { "diffusers_offload_nonblocking": OptionInfo(False, "Non-blocking move operations"), "offload_balanced_sep": OptionInfo("

Balanced Offload

", "", gr.HTML), "diffusers_offload_pre": OptionInfo(True, "Offload during pre-forward"), + "diffusers_offload_streams": OptionInfo(False, "Offload using streams"), "diffusers_offload_min_gpu_memory": OptionInfo(startup_offload_min_gpu, "Offload low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), "diffusers_offload_max_gpu_memory": OptionInfo(startup_offload_max_gpu, "Offload GPU high watermark", gr.Slider, {"minimum": 0.1, "maximum": 1, "step": 0.01 }), "diffusers_offload_max_cpu_memory": OptionInfo(0.90, "Offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False }), + "models_not_to_offload": OptionInfo("", "Model types not to offload"), "diffusers_offload_always": OptionInfo(startup_offload_always, "Modules to always offload"), "diffusers_offload_never": OptionInfo(startup_offload_never, "Modules to never offload"), "offload_group_sep": OptionInfo("

Group Offload

", "", gr.HTML), @@ -183,7 +191,7 @@ options_templates.update(options_section(('offload', "Model Offloading"), { })) options_templates.update(options_section(("quantization", "Model Quantization"), { - "models_not_to_quant": OptionInfo("", "List of model types not to quantize"), + "models_not_to_quant": OptionInfo("", "Model types not to quantize"), "sdnq_quantize_sep": OptionInfo("

SDNQ: SD.Next Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "TE", "LLM", "Control", "VAE"]}), @@ -248,8 +256,8 @@ options_templates.update(options_section(('vae_encoder', "Variational Auto Encod "sd_vae": OptionInfo("Automatic", "VAE model", gr.Dropdown, lambda: {"choices": shared_items.sd_vae_items()}, refresh=shared_items.refresh_vae_list), "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, {"choices": ['default', 'true', 'false']}), "no_half_vae": OptionInfo(False if not cmd_opts.use_openvino else True, "Full precision (--no-half-vae)"), - "diffusers_vae_slicing": OptionInfo(True, "VAE slicing", gr.Checkbox), - "diffusers_vae_tiling": OptionInfo(cmd_opts.lowvram or cmd_opts.medvram, "VAE tiling", gr.Checkbox), + "diffusers_vae_slicing": OptionInfo(cmd_opts.lowvram or cmd_opts.medvram, "VAE slicing", gr.Checkbox), + "diffusers_vae_tiling": OptionInfo(cmd_opts.lowvram, "VAE tiling", gr.Checkbox), "diffusers_vae_tile_size": OptionInfo(0, "VAE tile size", gr.Slider, {"minimum": 0, "maximum": 4096, "step": 8 }), "diffusers_vae_tile_overlap": OptionInfo(0.25, "VAE tile overlap", gr.Slider, {"minimum": 0, "maximum": 0.95, "step": 0.05 }), "remote_vae_type": OptionInfo('raw', "Remote VAE image type", gr.Dropdown, {"choices": ['raw', 'jpg', 'png']}), @@ -282,6 +290,8 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "cross_attention_sep": OptionInfo("

Cross Attention

", "", gr.HTML), "cross_attention_optimization": OptionInfo(startup_cross_attention, "Attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention()}), + "attention_": OptionInfo("

Cross Attention

", "", gr.HTML), + "attention_slicing": OptionInfo('Default', "Attention slicing", gr.Radio, {"choices": ['Default', 'Enabled', 'Disabled']}), "sdp_options": OptionInfo(startup_sdp_options, "SDP options", gr.CheckboxGroup, {"choices": startup_sdp_choices}), "xformers_options": OptionInfo(['Flash attention'], "xFormers options", gr.CheckboxGroup, {"choices": ['Flash attention'] }), "dynamic_attention_slice_rate": OptionInfo(0.5, "Dynamic Attention slicing rate in GB", gr.Slider, {"minimum": 0.01, "maximum": max(gpu_memory,4), "step": 0.01}), @@ -354,6 +364,14 @@ options_templates.update(options_section(('advanced', "Pipeline Modifiers"), { "pab_spacial_skip_start": OptionInfo(100, "PAB spacial skip start", gr.Slider, {"minimum": 0, "maximum": 1000, "step": 1}), "pab_spacial_skip_end": OptionInfo(800, "PAB spacial skip end", gr.Slider, {"minimum": 0, "maximum": 1000, "step": 1}), + "cache_dit_sep": OptionInfo("

Cache-DiT

", "", gr.HTML), + "cache_dit_enabled": OptionInfo(False, "Cache-DiT enabled"), + "cache_dit_calibrator": OptionInfo("None", "Cache-DiT calibrator", gr.Radio, {"choices": ["None", "TaylorSeer", "FoCa"]}), + "cache_dit_fcompute": OptionInfo(-1, "Cache-DiT F-compute blocks", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), + "cache_dit_bcompute": OptionInfo(-1, "Cache-DiT B-compute blocks", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), + "cache_dit_threshold": OptionInfo(-1, "Cache-DiT residual diff threshold", gr.Slider, {"minimum": -1.0, "maximum": 1.0, "step": 0.01}), + "cache_dit_warmup": OptionInfo(-1, "Cache-DiT warmup steps", gr.Slider, {"minimum": -1, "maximum": 50, "step": 1}), + "faster_cache__sep": OptionInfo("

Faster Cache

", "", gr.HTML), "faster_cache_enabled": OptionInfo(False, "FasterCache cache enabled"), "fc_spacial_skip_range": OptionInfo(2, "FasterCache spacial skip range", gr.Slider, {"minimum": 1, "maximum": 4, "step": 1}), @@ -453,7 +471,7 @@ options_templates.update(options_section(('system-paths', "System Paths"), { options_templates.update(options_section(('saving-images', "Image Options"), { "samples_save": OptionInfo(True, "Save all generated images"), - "keep_incomplete": OptionInfo(True, "Keep incomplete images"), + "keep_incomplete": OptionInfo(True, "Save interrupted images"), "samples_format": OptionInfo('jpg', 'File format', gr.Dropdown, {"choices": ["jpg", "png", "webp", "tiff", "jp2", "jxl"]}), "jpeg_quality": OptionInfo(90, "Image quality", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}), "img_max_size_mp": OptionInfo(1000, "Maximum image size (MP)", gr.Slider, {"minimum": 100, "maximum": 2000, "step": 1}), diff --git a/modules/shared_items.py b/modules/shared_items.py index 8eac5e070..f4d98e955 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -120,7 +120,6 @@ def list_crossattention(): "Scaled-Dot-Product", "xFormers", "Batch matrix-matrix", - "Split attention", "Dynamic Attention BMM" ] diff --git a/modules/styles.py b/modules/styles.py index ec3ca71e9..4ea4509a6 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -123,7 +123,7 @@ def apply_wildcards_to_prompt(prompt, all_wildcards, seed=-1, silent=False): if replaced and not silent: shared.log.debug(f'Apply wildcards: {replaced} path="{shared.opts.wildcards_dir}" type=style time={t1-t0:.2f}') if (len(replaced_file) > 0 or len(not_found) > 0) and not silent: - shared.log.debug(f'Apply wildcards: found={replaced_file} missing={not_found} path="{shared.opts.wildcards_dir}" type=file time={t2-t2:.2f} ') + shared.log.debug(f'Apply wildcards: found={replaced_file} missing={not_found} path="{shared.opts.wildcards_dir}" type=file seed={seed} time={t2-t2:.2f}') if old_state is not None: random.setstate(old_state) return prompt @@ -168,6 +168,7 @@ def apply_styles_to_extra(p, style: Style): params = [] settings = [] skipped = [] + for k, v in extra.items(): k = k.lower().replace(' ', '_') if k in name_map: # rename some fields @@ -322,16 +323,24 @@ class StyleDatabase: jobid = shared.state.begin('Styles') parsed_positive = [] parsed_negative = [] + random_state = random.getstate() + for i in range(len(prompts)): + if seeds[i]> 0: + random.seed(seeds[i]) prompt = prompts[i] prompt = apply_styles_to_prompt(prompt, [self.find_style(x).prompt for x in styles]) prompt = apply_wildcards_to_prompt(prompt, [self.find_style(x).wildcards for x in styles], seeds[i]) parsed_positive.append(prompt) for i in range(len(negatives)): + if seeds[i]> 0: + random.seed(seeds[i]) prompt = negatives[i] prompt = apply_styles_to_prompt(prompt, [self.find_style(x).negative_prompt for x in styles]) prompt = apply_wildcards_to_prompt(prompt, [self.find_style(x).wildcards for x in styles], seeds[i]) parsed_negative.append(prompt) + + random.setstate(random_state) shared.state.end(jobid) return parsed_positive, parsed_negative diff --git a/modules/txt2img.py b/modules/txt2img.py index 3397d4fe9..08d4f1bf2 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -14,6 +14,7 @@ def txt2img(id_task, state, vae_type, tiling, hidiffusion, detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, n_iter, batch_size, + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, clip_skip, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, @@ -55,6 +56,11 @@ def txt2img(id_task, state, batch_size=batch_size, n_iter=n_iter, steps=steps, + guidance_name=guidance_name, + guidance_scale=guidance_scale, + guidance_rescale=guidance_rescale, + guidance_start=guidance_start, + guidance_stop=guidance_stop, cfg_scale=cfg_scale, image_cfg_scale=image_cfg_scale, diffusers_guidance_rescale=diffusers_guidance_rescale, diff --git a/modules/ui.py b/modules/ui.py index d87f02c70..f9daddb35 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -28,6 +28,7 @@ plaintext_to_html = ui_common.plaintext_to_html # compatibility item infotext_to_html = ui_common.infotext_to_html # compatibility item create_sampler_and_steps_selection = ui_sections.create_sampler_and_steps_selection # compatibility item ui_system_tabs = None # required for system-info +interfaces = [] if not shared.cmd_opts.share and not shared.cmd_opts.listen: @@ -132,7 +133,8 @@ def create_ui(startup_timer = None): ui_extensions.create_ui() timer.startup.record("ui-extensions") - interfaces = [] + global interfaces # pylint: disable=global-statement + interfaces.clear() interfaces += [(txt2img_interface, "Text", "txt2img")] interfaces += [(img2img_interface, "Image", "img2img")] if control_interface is not None: diff --git a/modules/ui_caption.py b/modules/ui_caption.py index 86e2eade3..e0ef85933 100644 --- a/modules/ui_caption.py +++ b/modules/ui_caption.py @@ -28,6 +28,7 @@ def update_clip_params(*args): def create_ui(): + shared.log.debug('UI initialize: tab=caption') with gr.Row(equal_height=False, variant='compact', elem_classes="caption", elem_id="caption_tab"): with gr.Column(variant='compact', elem_id='interrogate_input'): with gr.Row(): diff --git a/modules/ui_common.py b/modules/ui_common.py index 1c5499cf7..7fa684190 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -5,7 +5,7 @@ import shutil import platform import subprocess import gradio as gr -from modules import call_queue, shared, errors, ui_sections, ui_symbols, ui_components, generation_parameters_copypaste, images, scripts_manager, script_callbacks, infotext +from modules import call_queue, shared, errors, ui_sections, ui_symbols, ui_components, generation_parameters_copypaste, images, scripts_manager, script_callbacks, infotext, processing folder_symbol = ui_symbols.folder @@ -19,8 +19,12 @@ def gr_show(visible=True): def update_generation_info(generation_info, html_info, img_index): try: + if len(generation_info) == 0 and processing.processed is not None: + generation_info = processing.processed.js() or {} + if len(generation_info) == 0: + return html_info, html_info generation_json = json.loads(generation_info) - if len(generation_json["infotexts"]) == 0: + if len(generation_json.get("infotexts", [])) == 0: return html_info, 'no infotexts found' if img_index == -1: img_index = 0 @@ -29,8 +33,8 @@ def update_generation_info(generation_info, html_info, img_index): info = generation_json["infotexts"][img_index] html_info_formatted = infotext_to_html(info) return html_info, html_info_formatted - except Exception: - pass + except Exception as e: + shared.log.trace(f'Update info: info="{generation_info}" {e}') return html_info, html_info @@ -238,7 +242,7 @@ def open_folder(result_gallery, gallery_index = 0): subprocess.Popen(["xdg-open", path]) # pylint: disable=consider-using-with -def create_output_panel(tabname, preview=True, prompt=None, height=None, transfer=True, scale=1): +def create_output_panel(tabname, preview=True, prompt=None, height=None, transfer=True, scale=1, result_info=None): with gr.Column(variant='panel', elem_id=f"{tabname}_results", scale=scale): with gr.Group(elem_id=f"{tabname}_gallery_container"): if tabname == "txt2img": @@ -286,10 +290,11 @@ def create_output_panel(tabname, preview=True, prompt=None, height=None, transfe generation_info = gr.Textbox(visible=False, elem_id=f'generation_info_{tabname}') generation_info_button = gr.Button(visible=False, elem_id=f"{tabname}_generation_info_button") + result_field = result_info or html_info_formatted generation_info_button.click(fn=update_generation_info, show_progress=False, _js="(x, y, z) => [x, y, selected_gallery_index()]", # triggered on gallery change from js inputs=[generation_info, html_info, html_info], - outputs=[html_info, html_info_formatted], + outputs=[html_info, result_field], ) save.click(fn=call_queue.wrap_gradio_call(save_files), show_progress=False, _js="(x, y, z, i) => [x, y, z, selected_gallery_index()]", @@ -354,11 +359,12 @@ def create_override_inputs(tab): # pylint: disable=unused-argument def reuse_seed(seed_component: gr.Number, reuse_button: gr.Button, subseed:bool=False): def reuse_click(selected_gallery_index): selected_gallery_index = int(selected_gallery_index) - from modules import processing if processing.processed is None: seed = -1 - elif selected_gallery_index >= len(processing.processed.all_seeds): - selected_gallery_index -= len(processing.processed.images) - len(processing.processed.all_seeds) # if we have more images than seeds it is likely the grid image + elif len(processing.processed.images) > len(processing.processed.all_seeds): # if we have more images than seeds it is likely the grid image + selected_gallery_index -= (len(processing.processed.images) - len(processing.processed.all_seeds)) + seed = processing.processed.all_seeds[selected_gallery_index] if not subseed else processing.processed.all_subseeds[selected_gallery_index] + elif selected_gallery_index <= len(processing.processed.all_seeds): seed = processing.processed.all_seeds[selected_gallery_index] if not subseed else processing.processed.all_subseeds[selected_gallery_index] elif len(processing.processed.all_seeds) > 0: seed = processing.processed.all_seeds[0] if not subseed else processing.processed.all_subseeds[0] diff --git a/modules/ui_control.py b/modules/ui_control.py index b75842aee..3f43701af 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -1,15 +1,9 @@ import os import time import gradio as gr -import matplotlib.pyplot as plt from modules.control import unit -from modules.control import processors # patrickvonplaten controlnet_aux -from modules.control.units import controlnet # lllyasviel ControlNet -from modules.control.units import xs # vislearn ControlNet-XS -from modules.control.units import lite # vislearn ControlNet-XS -from modules.control.units import t2iadapter # TencentARC T2I-Adapter -from modules.control.units import reference # reference pipeline -from modules import errors, shared, progress, ui_components, ui_symbols, ui_common, ui_sections, generation_parameters_copypaste, call_queue, scripts_manager, masking, images, processing_vae, timer # pylint: disable=ungrouped-imports +from modules import errors, shared, progress, generation_parameters_copypaste, call_queue, scripts_manager, masking, images, processing_vae, timer # pylint: disable=ungrouped-imports +from modules import ui_common, ui_sections, ui_guidance from modules import ui_control_helpers as helpers @@ -163,7 +157,8 @@ def create_ui(_blocks: gr.Blocks=None): mask_controls = masking.create_segment_ui() - vae_type, tiling, hidiffusion, cfg_scale, clip_skip, image_cfg_scale, guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_sections.create_advanced_inputs('control') + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_guidance.create_guidance_inputs('control') + vae_type, tiling, hidiffusion, clip_skip = ui_sections.create_advanced_inputs('control') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('control') with gr.Accordion(open=False, label="Video", elem_id="control_video", elem_classes=["small-accordion"]): @@ -217,7 +212,7 @@ def create_ui(_blocks: gr.Blocks=None): gr.HTML('Output

') with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-output') as output_tabs: with gr.Tab('Gallery', id='out-gallery'): - output_gallery, _output_gen_info, _output_html_info, _output_html_info_formatted, output_html_log = ui_common.create_output_panel("control", preview=False, prompt=prompt, height=gr_height) + output_gallery, _output_gen_info, _output_html_info, _output_html_info_formatted, output_html_log = ui_common.create_output_panel("control", preview=False, prompt=prompt, height=gr_height, result_info=result_txt) with gr.Tab('Image', id='out-image'): output_image = gr.Image(label="Output", show_label=False, type="pil", interactive=False, tool="editor", height=gr_height, elem_id='control_output_image', elem_classes=['control-image']) with gr.Tab('Video', id='out-video'): @@ -228,312 +223,9 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Tab('Preview', id='preview-image') as _tab_preview: preview_process = gr.Image(label="Preview", show_label=False, type="pil", interactive=False, height=gr_height, visible=True, elem_id='control_preview', elem_classes=['control-image']) - with gr.Accordion('Control elements', open=False, elem_id="control_elements"): - with gr.Tabs(elem_id='control-tabs') as _tabs_control_type: - with gr.Tab('ControlNet') as _tab_controlnet: - gr.HTML('ControlNet') - with gr.Row(): - extra_controls = [ - gr.Checkbox(label="Guess mode", value=False, scale=3), - ] - num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) - controlnet_ui_units = [] # list of hidable accordions - for i in range(max_units): - enabled = True if i==0 else False - with gr.Accordion(f'ControlNet unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: - with gr.Row(): - enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') - process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') - model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') - ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, f'controlnet_models_{i}_refresh') - control_mode = gr.Dropdown(label="CN Mode", choices=['default'], value='default', visible=False, elem_id=f'control_unit-{i}-mode') - model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=2.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') - control_start = gr.Slider(label="CN Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') - control_end = gr.Slider(label="CN End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') - control_tile = gr.Dropdown(label="CN Tiles", choices=[x.strip() for x in shared.opts.control_tiles.split(',') if 'x' in x], value='1x1', visible=False, elem_id=f'control_unit-{i}-tile') - reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'controlnet_unit-{i}-reset') - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'controlnet_unit-{i}-upload') - image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'controlnet_unit-{i}-reuse') - btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'controlnet_unit-{i}-preview') - image_preview = gr.Image(label="Input", type="pil", height=128, width=128, visible=False, interactive=True, show_label=False, show_download_button=False, container=False, elem_id=f'controlnet_unit-{i}-override') - controlnet_ui_units.append(unit_ui) - units.append(unit.Unit( - unit_type = 'controlnet', - index = i, - enabled = enabled, - result_txt = result_txt, - enabled_cb = enabled_cb, - reset_btn = reset_btn, - process_id = process_id, - model_id = model_id, - model_strength = model_strength, - preview_process = preview_process, - preview_btn = btn_preview, - image_upload = image_upload, - image_reuse = image_reuse, - image_preview = image_preview, - control_start = control_start, - control_end = control_end, - control_mode = control_mode, - control_tile = control_tile, - extra_controls = extra_controls, - ) - ) - if i == 0: - units[-1].enabled = True # enable first unit in group - num_controlnet_units.change(fn=helpers.display_units, inputs=[num_controlnet_units], outputs=controlnet_ui_units) - - with gr.Tab('T2I Adapter') as _tab_t2iadapter: - gr.HTML('T2I-Adapter') - with gr.Row(): - extra_controls = [ - gr.Slider(label="Control factor", minimum=0.0, maximum=1.0, step=0.05, value=1.0, scale=3), - ] - num_adapter_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) - adapter_ui_units = [] # list of hidable accordions - for i in range(max_units): - enabled = True if i==0 else False - with gr.Accordion(f'T2I-Adapter unit {i+1}', visible= i < num_adapter_units.value, elem_classes='control-unit') as unit_ui: - with gr.Row(): - enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') - process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') - model_id = gr.Dropdown(label="Adapter", choices=t2iadapter.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') - ui_common.create_refresh_button(model_id, t2iadapter.list_models, lambda: {"choices": t2iadapter.list_models(refresh=True)}, f'adapter_models_{i}_refresh') - model_strength = gr.Slider(label="T2I Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') - reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'adapter_unit-{i}-reset') - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'adapter_unit-{i}-upload') - image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'adapter_unit-{i}-reuse') - btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'adapter_unit-{i}-preview') - image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'adapter_unit-{i}-override') - adapter_ui_units.append(unit_ui) - units.append(unit.Unit( - unit_type = 't2i adapter', - index = i, - enabled = enabled, - result_txt = result_txt, - enabled_cb = enabled_cb, - reset_btn = reset_btn, - process_id = process_id, - model_id = model_id, - model_strength = model_strength, - preview_process = preview_process, - preview_btn = btn_preview, - image_upload = image_upload, - image_reuse = image_reuse, - image_preview = image_preview, - extra_controls = extra_controls, - ) - ) - if i == 0: - units[-1].enabled = True # enable first unit in group - num_adapter_units.change(fn=helpers.display_units, inputs=[num_adapter_units], outputs=adapter_ui_units) - - with gr.Tab('XS') as _tab_controlnetxs: - gr.HTML('ControlNet XS') - with gr.Row(): - extra_controls = [ - gr.Slider(label="Time embedding mix", minimum=0.0, maximum=1.0, step=0.05, value=0.0, scale=3) - ] - num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) - controlnetxs_ui_units = [] # list of hidable accordions - for i in range(max_units): - enabled = True if i==0 else False - with gr.Accordion(f'ControlNet-XS unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: - with gr.Row(): - enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') - process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') - model_id = gr.Dropdown(label="ControlNet-XS", choices=xs.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') - ui_common.create_refresh_button(model_id, xs.list_models, lambda: {"choices": xs.list_models(refresh=True)}, f'xs_models_{i}_refresh') - model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') - control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') - control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') - reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'controlnetxs_unit-{i}-reset') - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'controlnetxs_unit-{i}-upload') - image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'controlnetxs_unit-{i}-reuse') - btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'controlnetxs_unit-{i}-preview') - image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'controlnetxs_unit-{i}-override') - controlnetxs_ui_units.append(unit_ui) - units.append(unit.Unit( - unit_type = 'xs', - index = i, - enabled = enabled, - result_txt = result_txt, - enabled_cb = enabled_cb, - reset_btn = reset_btn, - process_id = process_id, - model_id = model_id, - model_strength = model_strength, - preview_process = preview_process, - preview_btn = btn_preview, - image_upload = image_upload, - image_reuse = image_reuse, - image_preview = image_preview, - control_start = control_start, - control_end = control_end, - extra_controls = extra_controls, - ) - ) - if i == 0: - units[-1].enabled = True # enable first unit in group - num_controlnet_units.change(fn=helpers.display_units, inputs=[num_controlnet_units], outputs=controlnetxs_ui_units) - - with gr.Tab('Lite') as _tab_lite: - gr.HTML('Control LLLite') - with gr.Row(): - extra_controls = [ - ] - num_lite_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) - lite_ui_units = [] # list of hidable accordions - for i in range(max_units): - enabled = True if i==0 else False - with gr.Accordion(f'Control-LLLite unit {i+1}', visible= i < num_lite_units.value, elem_classes='control-unit') as unit_ui: - with gr.Row(): - enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') - process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') - model_id = gr.Dropdown(label="Model", choices=lite.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') - ui_common.create_refresh_button(model_id, lite.list_models, lambda: {"choices": lite.list_models(refresh=True)}, f'lite_models_{i}_refresh') - model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') - reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'lite_unit-{i}-reset') - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'lite_unit-{i}-upload') - image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'lite_unit-{i}-reuse') - image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'lite_unit-{i}-override') - btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'lite_unit-{i}-preview') - lite_ui_units.append(unit_ui) - units.append(unit.Unit( - unit_type = 'lite', - index = i, - enabled = enabled, - result_txt = result_txt, - enabled_cb = enabled_cb, - reset_btn = reset_btn, - process_id = process_id, - model_id = model_id, - model_strength = model_strength, - preview_process = preview_process, - preview_btn = btn_preview, - image_upload = image_upload, - image_reuse = image_reuse, - image_preview = image_preview, - extra_controls = extra_controls, - ) - ) - if i == 0: - units[-1].enabled = True # enable first unit in group - num_lite_units.change(fn=helpers.display_units, inputs=[num_lite_units], outputs=lite_ui_units) - - with gr.Tab('Reference') as _tab_reference: - gr.HTML('ControlNet reference-only control') - with gr.Row(): - extra_controls = [ - gr.Radio(label="Reference context", choices=['Attention', 'Adain', 'Attention Adain'], value='Attention', interactive=True), - gr.Slider(label="Style fidelity", minimum=0.0, maximum=1.0, step=0.05, value=0.5, interactive=True), # prompt vs control importance - gr.Slider(label="Reference query weight", minimum=0.0, maximum=1.0, step=0.05, value=1.0, interactive=True), - gr.Slider(label="Reference adain weight", minimum=0.0, maximum=2.0, step=0.05, value=1.0, interactive=True), - ] - for i in range(1): # can only have one reference unit - enabled = True if i==0 else False - with gr.Accordion(f'Reference unit {i+1}', visible=True, elem_classes='control-unit') as unit_ui: - with gr.Row(): - enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') - model_id = gr.Dropdown(label="Reference", choices=reference.list_models(), value='Reference', visible=False, elem_id=f'control_unit-{i}-model_name') - model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, visible=False, elem_id=f'control_unit-{i}-strength') - reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'reference_unit-{i}-reset') - image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'reference_unit-{i}-upload') - image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'reference_unit-{i}-reuse') - image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'reference_unit-{i}-override') - btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'reference_unit-{i}-preview') - units.append(unit.Unit( - unit_type = 'reference', - index = i, - enabled = enabled, - result_txt = result_txt, - enabled_cb = enabled_cb, - reset_btn = reset_btn, - process_id = process_id, - model_id = model_id, - model_strength = model_strength, - preview_process = preview_process, - preview_btn = btn_preview, - image_upload = image_upload, - image_reuse = image_reuse, - image_preview = image_preview, - extra_controls = extra_controls, - ) - ) - if i == 0: - units[-1].enabled = True # enable first unit in group - - with gr.Accordion('Control settings', open=False, elem_classes=['control-settings']) as _tab_settings: - with gr.Group(elem_classes=['processor-group']): - settings = [] - with gr.Accordion('Global', open=True, elem_classes=['processor-settings']): - control_hires = gr.Checkbox(label="Use control during hires", value=shared.opts.control_hires, elem_id='control_hires') - def set_control_hires(value): - shared.opts.control_active = value - control_hires.change(fn=set_control_hires, inputs=[control_hires], outputs=[]) - control_max_units = gr.Slider(label="Maximum units", minimum=1, maximum=10, step=1, value=shared.opts.control_max_units, elem_id='control_max_units') - def set_control_max_units(value): - shared.opts.control_max_units = value - control_max_units.change(fn=set_control_max_units, inputs=[control_max_units], outputs=[]) - control_tiles = gr.Textbox(label="Tiling options", value=shared.opts.control_tiles, elem_id='control_tiles') - def set_control_tiles(value): - shared.opts.control_tiles = value - control_tiles.change(fn=set_control_tiles, inputs=[control_tiles], outputs=[]) - control_move_processor = gr.Checkbox(label="Move processor to CPU after use", value=shared.opts.control_move_processor, elem_id='control_move_processor') - def set_control_move_processor(value): - shared.opts.control_move_processor = value - control_move_processor.change(fn=set_control_move_processor, inputs=[control_move_processor], outputs=[]) - control_unload_processor = gr.Checkbox(label="Unload processor after use", value=shared.opts.control_unload_processor, elem_id='control_unload_processor') - def set_control_unload_processor(value): - shared.opts.control_unload_processor = value - control_unload_processor.change(fn=set_control_unload_processor, inputs=[control_unload_processor], outputs=[]) - - with gr.Accordion('HED', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Scribble", value=False)) - with gr.Accordion('Midas depth', open=True, elem_classes=['processor-settings']): - settings.append(gr.Slider(label="Background threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1)) - settings.append(gr.Checkbox(label="Depth and normal", value=False)) - with gr.Accordion('MLSD', open=True, elem_classes=['processor-settings']): - settings.append(gr.Slider(label="Score threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1)) - settings.append(gr.Slider(label="Distance threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1)) - with gr.Accordion('OpenBody', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Body", value=True)) - settings.append(gr.Checkbox(label="Hands", value=False)) - settings.append(gr.Checkbox(label="Face", value=False)) - with gr.Accordion('PidiNet', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Scribble", value=False)) - settings.append(gr.Checkbox(label="Apply filter", value=False)) - with gr.Accordion('LineArt', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Coarse", value=False)) - with gr.Accordion('Leres Depth', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Boost", value=False)) - settings.append(gr.Slider(label="Near threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.0)) - settings.append(gr.Slider(label="Depth threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.0)) - with gr.Accordion('MediaPipe Face', open=True, elem_classes=['processor-settings']): - settings.append(gr.Slider(label="Max faces", minimum=1, maximum=10, step=1, value=1)) - settings.append(gr.Slider(label="Face confidence", minimum=0.0, maximum=1.0, step=0.01, value=0.5)) - with gr.Accordion('Canny', open=True, elem_classes=['processor-settings']): - settings.append(gr.Slider(label="Low threshold", minimum=0, maximum=1000, step=1, value=100)) - settings.append(gr.Slider(label="High threshold", minimum=0, maximum=1000, step=1, value=200)) - with gr.Accordion('DWPose', open=True, elem_classes=['processor-settings']): - settings.append(gr.Radio(label="Pose Model", choices=['Tiny', 'Medium', 'Large'], value='Tiny')) - settings.append(gr.Slider(label="Pose confidence", minimum=0.0, maximum=1.0, step=0.01, value=0.3)) - with gr.Accordion('SegmentAnything', open=True, elem_classes=['processor-settings']): - settings.append(gr.Radio(label="Segment Model", choices=['Base', 'Large'], value='Base')) - with gr.Accordion('Edge', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Parameter free", value=True)) - settings.append(gr.Radio(label="Edge mode", choices=['edge', 'gradient'], value='edge')) - with gr.Accordion('Zoe Depth', open=True, elem_classes=['processor-settings']): - settings.append(gr.Checkbox(label="Gamma corrected", value=False)) - with gr.Accordion('Marigold Depth', open=True, elem_classes=['processor-settings']): - settings.append(gr.Dropdown(label="Color map", choices=['None'] + plt.colormaps(), value='None')) - settings.append(gr.Slider(label="Denoising steps", minimum=1, maximum=99, step=1, value=10)) - settings.append(gr.Slider(label="Ensemble size", minimum=1, maximum=99, step=1, value=10)) - with gr.Accordion('Depth Anything', open=True, elem_classes=['processor-settings']): - settings.append(gr.Dropdown(label="Depth map", choices=['none'] + masking.COLORMAP, value='inferno')) - for setting in settings: - setting.change(fn=processors.update_settings, inputs=settings, outputs=[]) + from modules.ui_control_elements import create_ui_elements + create_ui_elements(units, result_txt, preview_process) with gr.Row(elem_id="control_script_container"): input_script_args = scripts_manager.scripts_current.setup_ui(parent='control', accordion=True) @@ -583,7 +275,8 @@ def create_ui(_blocks: gr.Blocks=None): prompt, negative, styles, steps, sampler_index, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, - cfg_scale, clip_skip, image_cfg_scale, guidance_rescale, pag_scale, pag_adaptive, cfg_end, vae_type, tiling, hidiffusion, + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, + cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, vae_type, tiling, hidiffusion, detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio, resize_mode_before, resize_name_before, resize_context_before, width_before, height_before, scale_by_before, selected_scale_tab_before, @@ -663,14 +356,19 @@ def create_ui(_blocks: gr.Blocks=None): (mask_controls[4], "Mask erode"), (mask_controls[5], "Mask dilate"), (mask_controls[6], "Mask auto"), + # guidance + (guidance_name, "Guidance"), + (guidance_scale, "Guidance scale"), + (guidance_rescale, "Guidance rescale"), + (guidance_start, "Guidance start"), + (guidance_stop, "Guidance stop"), # advanced - (cfg_scale, "Guidance scale"), (cfg_scale, "CFG scale"), (cfg_end, "CFG end"), (clip_skip, "Clip skip"), (image_cfg_scale, "Image CFG scale"), (image_cfg_scale, "Hires CFG scale"), - (guidance_rescale, "CFG rescale"), + (diffusers_guidance_rescale, "CFG rescale"), (vae_type, "VAE type"), (tiling, "Tiling"), (hidiffusion, "HiDiffusion"), diff --git a/modules/ui_control_elements.py b/modules/ui_control_elements.py new file mode 100644 index 000000000..21f7b4ca5 --- /dev/null +++ b/modules/ui_control_elements.py @@ -0,0 +1,321 @@ +import gradio as gr +import matplotlib.pyplot as plt +from modules.control import unit +from modules.control import processors # patrickvonplaten controlnet_aux +from modules.control.units import controlnet # lllyasviel ControlNet +from modules.control.units import xs # vislearn ControlNet-XS +from modules.control.units import lite # vislearn ControlNet-XS +from modules.control.units import t2iadapter # TencentARC T2I-Adapter +from modules.control.units import reference # reference pipeline +from modules import shared, ui_components, ui_symbols, ui_common, masking # pylint: disable=ungrouped-imports +from modules import ui_control_helpers as helpers + + +def create_ui_elements(units, result_txt, preview_process): + max_units = shared.opts.control_max_units + with gr.Accordion('Control elements', open=False, elem_id="control_elements"): + with gr.Tabs(elem_id='control-tabs') as _tabs_control_type: + + with gr.Tab('ControlNet') as _tab_controlnet: + gr.HTML('ControlNet') + with gr.Row(): + extra_controls = [ + gr.Checkbox(label="Guess mode", value=False, scale=3), + ] + num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) + controlnet_ui_units = [] # list of hidable accordions + for i in range(max_units): + enabled = True if i==0 else False + with gr.Accordion(f'ControlNet unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: + with gr.Row(): + enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') + process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') + model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') + ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, f'controlnet_models_{i}_refresh') + control_mode = gr.Dropdown(label="CN Mode", choices=['default'], value='default', visible=False, elem_id=f'control_unit-{i}-mode') + model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=2.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') + control_start = gr.Slider(label="CN Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') + control_end = gr.Slider(label="CN End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') + control_tile = gr.Dropdown(label="CN Tiles", choices=[x.strip() for x in shared.opts.control_tiles.split(',') if 'x' in x], value='1x1', visible=False, elem_id=f'control_unit-{i}-tile') + reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'controlnet_unit-{i}-reset') + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'controlnet_unit-{i}-upload') + image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'controlnet_unit-{i}-reuse') + btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'controlnet_unit-{i}-preview') + image_preview = gr.Image(label="Input", type="pil", height=128, width=128, visible=False, interactive=True, show_label=False, show_download_button=False, container=False, elem_id=f'controlnet_unit-{i}-override') + controlnet_ui_units.append(unit_ui) + units.append(unit.Unit( + unit_type = 'controlnet', + index = i, + enabled = enabled, + result_txt = result_txt, + enabled_cb = enabled_cb, + reset_btn = reset_btn, + process_id = process_id, + model_id = model_id, + model_strength = model_strength, + preview_process = preview_process, + preview_btn = btn_preview, + image_upload = image_upload, + image_reuse = image_reuse, + image_preview = image_preview, + control_start = control_start, + control_end = control_end, + control_mode = control_mode, + control_tile = control_tile, + extra_controls = extra_controls, + ) + ) + if i == 0: + units[-1].enabled = True # enable first unit in group + num_controlnet_units.change(fn=helpers.display_units, inputs=[num_controlnet_units], outputs=controlnet_ui_units) + + with gr.Tab('T2I Adapter') as _tab_t2iadapter: + gr.HTML('T2I-Adapter') + with gr.Row(): + extra_controls = [ + gr.Slider(label="Control factor", minimum=0.0, maximum=1.0, step=0.05, value=1.0, scale=3), + ] + num_adapter_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) + adapter_ui_units = [] # list of hidable accordions + for i in range(max_units): + enabled = True if i==0 else False + with gr.Accordion(f'T2I-Adapter unit {i+1}', visible= i < num_adapter_units.value, elem_classes='control-unit') as unit_ui: + with gr.Row(): + enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') + process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') + model_id = gr.Dropdown(label="Adapter", choices=t2iadapter.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') + ui_common.create_refresh_button(model_id, t2iadapter.list_models, lambda: {"choices": t2iadapter.list_models(refresh=True)}, f'adapter_models_{i}_refresh') + model_strength = gr.Slider(label="T2I Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') + reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'adapter_unit-{i}-reset') + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'adapter_unit-{i}-upload') + image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'adapter_unit-{i}-reuse') + btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'adapter_unit-{i}-preview') + image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'adapter_unit-{i}-override') + adapter_ui_units.append(unit_ui) + units.append(unit.Unit( + unit_type = 't2i adapter', + index = i, + enabled = enabled, + result_txt = result_txt, + enabled_cb = enabled_cb, + reset_btn = reset_btn, + process_id = process_id, + model_id = model_id, + model_strength = model_strength, + preview_process = preview_process, + preview_btn = btn_preview, + image_upload = image_upload, + image_reuse = image_reuse, + image_preview = image_preview, + extra_controls = extra_controls, + ) + ) + if i == 0: + units[-1].enabled = True # enable first unit in group + num_adapter_units.change(fn=helpers.display_units, inputs=[num_adapter_units], outputs=adapter_ui_units) + + with gr.Tab('XS') as _tab_controlnetxs: + gr.HTML('ControlNet XS') + with gr.Row(): + extra_controls = [ + gr.Slider(label="Time embedding mix", minimum=0.0, maximum=1.0, step=0.05, value=0.0, scale=3) + ] + num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) + controlnetxs_ui_units = [] # list of hidable accordions + for i in range(max_units): + enabled = True if i==0 else False + with gr.Accordion(f'ControlNet-XS unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: + with gr.Row(): + enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') + process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') + model_id = gr.Dropdown(label="ControlNet-XS", choices=xs.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') + ui_common.create_refresh_button(model_id, xs.list_models, lambda: {"choices": xs.list_models(refresh=True)}, f'xs_models_{i}_refresh') + model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') + control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') + control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') + reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'controlnetxs_unit-{i}-reset') + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'controlnetxs_unit-{i}-upload') + image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'controlnetxs_unit-{i}-reuse') + btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'controlnetxs_unit-{i}-preview') + image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'controlnetxs_unit-{i}-override') + controlnetxs_ui_units.append(unit_ui) + units.append(unit.Unit( + unit_type = 'xs', + index = i, + enabled = enabled, + result_txt = result_txt, + enabled_cb = enabled_cb, + reset_btn = reset_btn, + process_id = process_id, + model_id = model_id, + model_strength = model_strength, + preview_process = preview_process, + preview_btn = btn_preview, + image_upload = image_upload, + image_reuse = image_reuse, + image_preview = image_preview, + control_start = control_start, + control_end = control_end, + extra_controls = extra_controls, + ) + ) + if i == 0: + units[-1].enabled = True # enable first unit in group + num_controlnet_units.change(fn=helpers.display_units, inputs=[num_controlnet_units], outputs=controlnetxs_ui_units) + + with gr.Tab('Lite') as _tab_lite: + gr.HTML('Control LLLite') + with gr.Row(): + extra_controls = [ + ] + num_lite_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) + lite_ui_units = [] # list of hidable accordions + for i in range(max_units): + enabled = True if i==0 else False + with gr.Accordion(f'Control-LLLite unit {i+1}', visible= i < num_lite_units.value, elem_classes='control-unit') as unit_ui: + with gr.Row(): + enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') + process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') + model_id = gr.Dropdown(label="Model", choices=lite.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') + ui_common.create_refresh_button(model_id, lite.list_models, lambda: {"choices": lite.list_models(refresh=True)}, f'lite_models_{i}_refresh') + model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') + reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'lite_unit-{i}-reset') + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'lite_unit-{i}-upload') + image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'lite_unit-{i}-reuse') + image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'lite_unit-{i}-override') + btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'lite_unit-{i}-preview') + lite_ui_units.append(unit_ui) + units.append(unit.Unit( + unit_type = 'lite', + index = i, + enabled = enabled, + result_txt = result_txt, + enabled_cb = enabled_cb, + reset_btn = reset_btn, + process_id = process_id, + model_id = model_id, + model_strength = model_strength, + preview_process = preview_process, + preview_btn = btn_preview, + image_upload = image_upload, + image_reuse = image_reuse, + image_preview = image_preview, + extra_controls = extra_controls, + ) + ) + if i == 0: + units[-1].enabled = True # enable first unit in group + num_lite_units.change(fn=helpers.display_units, inputs=[num_lite_units], outputs=lite_ui_units) + + with gr.Tab('Reference') as _tab_reference: + gr.HTML('ControlNet reference-only control') + with gr.Row(): + extra_controls = [ + gr.Radio(label="Reference context", choices=['Attention', 'Adain', 'Attention Adain'], value='Attention', interactive=True), + gr.Slider(label="Style fidelity", minimum=0.0, maximum=1.0, step=0.05, value=0.5, interactive=True), # prompt vs control importance + gr.Slider(label="Reference query weight", minimum=0.0, maximum=1.0, step=0.05, value=1.0, interactive=True), + gr.Slider(label="Reference adain weight", minimum=0.0, maximum=2.0, step=0.05, value=1.0, interactive=True), + ] + for i in range(1): # can only have one reference unit + enabled = True if i==0 else False + with gr.Accordion(f'Reference unit {i+1}', visible=True, elem_classes='control-unit') as unit_ui: + with gr.Row(): + enabled_cb = gr.Checkbox(enabled, label='Active', container=False, show_label=True, elem_id=f'control_unit-{i}-enabled') + model_id = gr.Dropdown(label="Reference", choices=reference.list_models(), value='Reference', visible=False, elem_id=f'control_unit-{i}-model_name') + model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0, visible=False, elem_id=f'control_unit-{i}-strength') + reset_btn = ui_components.ToolButton(value=ui_symbols.reset, elem_id=f'reference_unit-{i}-reset') + image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool'], elem_id=f'reference_unit-{i}-upload') + image_reuse= ui_components.ToolButton(value=ui_symbols.reuse, elem_id=f'reference_unit-{i}-reuse') + image_preview = gr.Image(label="Input", show_label=False, type="pil", interactive=False, height=128, width=128, visible=False, elem_id=f'reference_unit-{i}-override') + btn_preview= ui_components.ToolButton(value=ui_symbols.preview, elem_id=f'reference_unit-{i}-preview') + units.append(unit.Unit( + unit_type = 'reference', + index = i, + enabled = enabled, + result_txt = result_txt, + enabled_cb = enabled_cb, + reset_btn = reset_btn, + process_id = process_id, + model_id = model_id, + model_strength = model_strength, + preview_process = preview_process, + preview_btn = btn_preview, + image_upload = image_upload, + image_reuse = image_reuse, + image_preview = image_preview, + extra_controls = extra_controls, + ) + ) + if i == 0: + units[-1].enabled = True # enable first unit in group + + with gr.Accordion('Control settings', open=False, elem_classes=['control-settings']) as _tab_settings: + with gr.Group(elem_classes=['processor-group']): + settings = [] + with gr.Accordion('Global', open=True, elem_classes=['processor-settings']): + control_hires = gr.Checkbox(label="Use control during hires", value=shared.opts.control_hires, elem_id='control_hires') + def set_control_hires(value): + shared.opts.control_active = value + control_hires.change(fn=set_control_hires, inputs=[control_hires], outputs=[]) + control_max_units = gr.Slider(label="Maximum units", minimum=1, maximum=10, step=1, value=shared.opts.control_max_units, elem_id='control_max_units') + def set_control_max_units(value): + shared.opts.control_max_units = value + control_max_units.change(fn=set_control_max_units, inputs=[control_max_units], outputs=[]) + control_tiles = gr.Textbox(label="Tiling options", value=shared.opts.control_tiles, elem_id='control_tiles') + def set_control_tiles(value): + shared.opts.control_tiles = value + control_tiles.change(fn=set_control_tiles, inputs=[control_tiles], outputs=[]) + control_move_processor = gr.Checkbox(label="Move processor to CPU after use", value=shared.opts.control_move_processor, elem_id='control_move_processor') + def set_control_move_processor(value): + shared.opts.control_move_processor = value + control_move_processor.change(fn=set_control_move_processor, inputs=[control_move_processor], outputs=[]) + control_unload_processor = gr.Checkbox(label="Unload processor after use", value=shared.opts.control_unload_processor, elem_id='control_unload_processor') + def set_control_unload_processor(value): + shared.opts.control_unload_processor = value + control_unload_processor.change(fn=set_control_unload_processor, inputs=[control_unload_processor], outputs=[]) + + with gr.Accordion('HED', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Scribble", value=False)) + with gr.Accordion('Midas depth', open=True, elem_classes=['processor-settings']): + settings.append(gr.Slider(label="Background threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1)) + settings.append(gr.Checkbox(label="Depth and normal", value=False)) + with gr.Accordion('MLSD', open=True, elem_classes=['processor-settings']): + settings.append(gr.Slider(label="Score threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1)) + settings.append(gr.Slider(label="Distance threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.1)) + with gr.Accordion('OpenBody', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Body", value=True)) + settings.append(gr.Checkbox(label="Hands", value=False)) + settings.append(gr.Checkbox(label="Face", value=False)) + with gr.Accordion('PidiNet', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Scribble", value=False)) + settings.append(gr.Checkbox(label="Apply filter", value=False)) + with gr.Accordion('LineArt', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Coarse", value=False)) + with gr.Accordion('Leres Depth', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Boost", value=False)) + settings.append(gr.Slider(label="Near threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.0)) + settings.append(gr.Slider(label="Depth threshold", minimum=0.0, maximum=1.0, step=0.01, value=0.0)) + with gr.Accordion('MediaPipe Face', open=True, elem_classes=['processor-settings']): + settings.append(gr.Slider(label="Max faces", minimum=1, maximum=10, step=1, value=1)) + settings.append(gr.Slider(label="Face confidence", minimum=0.0, maximum=1.0, step=0.01, value=0.5)) + with gr.Accordion('Canny', open=True, elem_classes=['processor-settings']): + settings.append(gr.Slider(label="Low threshold", minimum=0, maximum=1000, step=1, value=100)) + settings.append(gr.Slider(label="High threshold", minimum=0, maximum=1000, step=1, value=200)) + with gr.Accordion('DWPose', open=True, elem_classes=['processor-settings']): + settings.append(gr.Radio(label="Pose Model", choices=['Tiny', 'Medium', 'Large'], value='Tiny')) + settings.append(gr.Slider(label="Pose confidence", minimum=0.0, maximum=1.0, step=0.01, value=0.3)) + with gr.Accordion('SegmentAnything', open=True, elem_classes=['processor-settings']): + settings.append(gr.Radio(label="Segment Model", choices=['Base', 'Large'], value='Base')) + with gr.Accordion('Edge', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Parameter free", value=True)) + settings.append(gr.Radio(label="Edge mode", choices=['edge', 'gradient'], value='edge')) + with gr.Accordion('Zoe Depth', open=True, elem_classes=['processor-settings']): + settings.append(gr.Checkbox(label="Gamma corrected", value=False)) + with gr.Accordion('Marigold Depth', open=True, elem_classes=['processor-settings']): + settings.append(gr.Dropdown(label="Color map", choices=['None'] + plt.colormaps(), value='None')) + settings.append(gr.Slider(label="Denoising steps", minimum=1, maximum=99, step=1, value=10)) + settings.append(gr.Slider(label="Ensemble size", minimum=1, maximum=99, step=1, value=10)) + with gr.Accordion('Depth Anything', open=True, elem_classes=['processor-settings']): + settings.append(gr.Dropdown(label="Depth map", choices=['none'] + masking.COLORMAP, value='inferno')) + for setting in settings: + setting.change(fn=processors.update_settings, inputs=settings, outputs=[]) diff --git a/modules/ui_control_helpers.py b/modules/ui_control_helpers.py index a04512863..c5757c56f 100644 --- a/modules/ui_control_helpers.py +++ b/modules/ui_control_helpers.py @@ -24,7 +24,7 @@ def initialize(): from modules.control.units import xs # vislearn ControlNet-XS from modules.control.units import lite # vislearn ControlNet-XS from modules.control.units import t2iadapter # TencentARC T2I-Adapter - shared.log.debug(f'UI initialize: control models="{shared.opts.control_dir}"') + shared.log.debug(f'UI initialize: tab=control models="{shared.opts.control_dir}"') controlnet.cache_dir = os.path.join(shared.opts.control_dir, 'controlnet') xs.cache_dir = os.path.join(shared.opts.control_dir, 'xs') lite.cache_dir = os.path.join(shared.opts.control_dir, 'lite') diff --git a/modules/ui_docs.py b/modules/ui_docs.py index d8b4e3dce..8b1381ce8 100644 --- a/modules/ui_docs.py +++ b/modules/ui_docs.py @@ -275,6 +275,7 @@ def create_ui_docs(): def create_ui(): + log.debug('UI initialize: tab=info') with gr.Tabs(elem_id="tabs_info"): with gr.TabItem("Docs", id="docs", elem_id="system_tab_docs"): create_ui_docs() diff --git a/modules/ui_extensions.py b/modules/ui_extensions.py index 28d792393..43158211a 100644 --- a/modules/ui_extensions.py +++ b/modules/ui_extensions.py @@ -430,6 +430,7 @@ def create_html(search_text, sort_column): def create_ui(): + shared.log.debug('UI initialize: tab=extensions') extensions_disable_all = gr.Radio(label="Disable all extensions", choices=["none", "user", "all"], value=shared.opts.disable_all_extensions, elem_id="extensions_disable_all", visible=False) extensions_disabled_list = gr.Textbox(elem_id="extensions_disabled_list", visible=False, container=False) extensions_update_list = gr.Textbox(elem_id="extensions_update_list", visible=False, container=False) diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 645dab049..c60c04c86 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -828,6 +828,10 @@ def create_ui(container, button_parent, tabname, skip_indexing = False): if is_valid: stat_size, stat_mtime = modelstats.stat(item.filename) + if hasattr(item, 'size') and item.size > 0: + stat_size = item.size + if hasattr(item, 'mtime') and item.mtime is not None: + stat_mtime = item.mtime desc = item.description fullinfo = shared.readfile(os.path.splitext(item.filename)[0] + '.json', silent=True) if 'modelVersions' in fullinfo: # sanitize massive objects diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index 1d320f760..ae4c98422 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -2,11 +2,13 @@ import os import html import json import concurrent +from datetime import datetime from modules import shared, ui_extra_networks, sd_models, modelstats, paths version_map = { "QwenEdit": "Qwen", + "QwenEditPlus": "Qwen", "Flux.1 D": "Flux", "Flux.1 S": "Flux", "FluxKontext": "Flux", @@ -14,6 +16,8 @@ version_map = { "SDXL Hyper": "SD XL", "StableDiffusion3": "SD 3", "StableDiffusionXL": "SD XL", + "WanToVideo": "Wan", + "WanVACE": "Wan", } class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): @@ -42,8 +46,16 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): continue preview = v.get('preview', v['path']) preview_file = self.find_preview_file(os.path.join(paths.reference_path, preview)) - _size, mtime = modelstats.stat(preview_file) name = os.path.normpath(os.path.join(paths.reference_path, k)).replace('\\', '/') + size = int(float(v.get('size', 0)) * 1024 * 1024 * 1024) + mtime = v.get('date', None) + if mtime is None: + _size, mtime = modelstats.stat(preview_file) + else: + try: + mtime = datetime.strptime(mtime, '%Y %B') # 2025 January + except Exception: + _size, mtime = modelstats.stat(preview_file) yield { "type": 'Model', "name": name, @@ -54,7 +66,7 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): "onclick": '"' + html.escape(f"selectReference({json.dumps(url)})") + '"', "hash": None, "mtime": mtime, - "size": 0, + "size": size, "info": {}, "metadata": {}, "description": v.get('desc', ''), diff --git a/modules/ui_extra_networks_lora.py b/modules/ui_extra_networks_lora.py index d390fe914..6b9705674 100644 --- a/modules/ui_extra_networks_lora.py +++ b/modules/ui_extra_networks_lora.py @@ -85,7 +85,7 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): "metadata": json.dumps(l.metadata, indent=4) if l.metadata else None, "mtime": mtime, "size": size, - "version": version.get("baseModel", l.sd_version) if info else l.sd_version, + "version": version.get("baseModel", l.sd_version), "info": info, "description": self.find_description(l.filename, info), "tags": self.get_tags(l, info, version), diff --git a/modules/ui_gallery.py b/modules/ui_gallery.py index e6cb3f297..283b720a8 100644 --- a/modules/ui_gallery.py +++ b/modules/ui_gallery.py @@ -54,6 +54,7 @@ def read_media(fn): def create_ui(): + shared.log.debug('UI initialize: tab=gallery') with gr.Blocks() as tab: with gr.Row(elem_id='tab-gallery-sort-buttons'): sort_buttons = [] diff --git a/modules/ui_guidance.py b/modules/ui_guidance.py new file mode 100644 index 000000000..3f3b3f8f3 --- /dev/null +++ b/modules/ui_guidance.py @@ -0,0 +1,125 @@ +import gradio as gr +from modules import shared, modular_guiders +from modules import ui_symbols, ui_components + + +def create_guidance_inputs(tab): + with gr.Accordion(open=False, label='Guidance', elem_id=f"{tab}_guidance", elem_classes=["small-accordion"]): + with gr.Group(): + + with gr.Row(elem_id=f"{tab}_guider_row", elem_classes=['flexbox'], visible=shared.opts.model_modular_enable): + guidance_name = gr.Dropdown(choices=list(modular_guiders.guiders.keys()), value='Default', label='Guider', elem_id=f"{tab}_guider") + guidance_btn = ui_components.ToolButton(value=ui_symbols.book, elem_id=f"{tab}_guider_docs") + guidance_btn.click(fn=None, _js='getGuidanceDocs', inputs=[guidance_name], outputs=[]) + with gr.Row(visible=shared.opts.model_modular_enable): + guidance_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Guidance scale', value=6.0, elem_id=f"{tab}_guidance_scale") + guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.0, elem_id=f"{tab}_guidance_rescale") + with gr.Row(visible=shared.opts.model_modular_enable): + guidance_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance start', value=0.0, elem_id=f"{tab}_guidance_start") + guidance_stop = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='Guidance stop', value=1.0, elem_id=f"{tab}_guidance_stop") + guidance_args = [guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop] + + lsc_group = gr.Accordion(open=False, label='Layer skip guidance', elem_classes=["small-accordion"], visible=shared.opts.model_modular_enable) + with lsc_group: + with gr.Row(): + guidance_lsc_enabled = gr.Checkbox(label='Enable LayerSkipConfig', value=False) + guidance_lsc_label = gr.Label(value='LSC: LayerSkipConfig', elem_id=f"{tab}_lsc_label", visible=False) + guidance_lsc_btn = ui_components.ToolButton(value=ui_symbols.book, elem_id=f"{tab}_lsc_docs", elem_classes=["guidance-docs"]) + guidance_lsc_btn.click(fn=None, _js='getGuidanceDocs', inputs=[guidance_lsc_label], outputs=[]) + with gr.Row(): + guidance_lsc_indices = gr.Textbox(label='LSC layer indices', value='1, 2, 3', placeholder='Comma-separated layer indices to skip') + with gr.Row(): + guidance_lsc_fqn = gr.Textbox(label='LSC fully qualified name', value='transformer_blocks', placeholder='Fully qualified name of the layer stack') + with gr.Row(): + guidance_lsc_skip_attention = gr.Checkbox(label='LSC skip attention blocks', value=True) + guidance_lsc_skip_ff = gr.Checkbox(label='LSC skip feed-forward blocks', value=True) + guidance_lsc_skip_attention_scores = gr.Checkbox(label='LSC skip attention scores', value=False) + with gr.Row(): + guidance_lsc_dropout = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='LSC dropout rate', value=1.0) + lsc_args = [guidance_lsc_enabled, guidance_lsc_indices, guidance_lsc_fqn, guidance_lsc_skip_attention, guidance_lsc_skip_ff, guidance_lsc_skip_attention_scores, guidance_lsc_dropout] + + auto_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with auto_group: + guidance_auto_dropout = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='AutoGuidance dropout', value=0.1) + guidance_auto_layers = gr.Textbox(label='AutoGuidance layers', value='7, 8, 9', placeholder='Comma-separated layer indices, e.g. 7,8,9') + guidance_auto_config = gr.Dropdown(choices=[None, 'config1', 'config2'], value=None, label='AutoGuidance config') + guidance_auto_args = [guidance_auto_dropout, guidance_auto_layers, guidance_auto_config] + + zero_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with zero_group: + guidance_zero_init_steps = gr.Slider(minimum=0, maximum=10, step=1, label='ZeroStar init steps', value=1) + guidance_zero_args = [guidance_zero_init_steps] + + pag_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with pag_group: + guidance_pag_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.05, label='PAG scale', value=2.8) + guidance_pag_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='PAG start', value=0.01) + guidance_pag_stop = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='PAG stop', value=0.2) + guidance_pag_layers = gr.Textbox(label='PAG layers', value='7, 8, 9', placeholder='Comma-separated layer indices, e.g. 7,8,9') + guidance_pag_config = gr.Dropdown(choices=[None, 'config1', 'config2'], value=None, label='PAG config') + guidance_pag_args = [guidance_pag_scale, guidance_pag_start, guidance_pag_stop, guidance_pag_layers, guidance_pag_config] + + apg_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with apg_group: + guidance_apg_momentum = gr.Slider(minimum=-1.0, maximum=1.0, step=0.05, label='APG momentum', value=-1.0) + guidance_apg_rescale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='APG rescale', value=15.0) + guidance_apg_args = [guidance_apg_momentum, guidance_apg_rescale] + + slg_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with slg_group: + guidance_slg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='SLG scale', value=2.8) + guidance_slg_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='SLG start', value=0.01) + guidance_slg_stop = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='SLG stop', value=0.2) + guidance_slg_layers = gr.Textbox(label='SLG layers', value='7, 8, 9', placeholder='Comma-separated layer indices, e.g. 7,8,9') + guidance_slg_config = gr.Dropdown(choices=[None, 'config1', 'config2'], value=None, label='SLG config') + guidance_slg_args = [guidance_slg_scale, guidance_slg_start, guidance_slg_stop, guidance_slg_layers, guidance_slg_config] + + seg_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with seg_group: + guidance_seg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='SEG scale', value=3.0) + guidance_seg_blur_sigma = gr.Number(label='SEG blur sigma', value=9999999.0) + guidance_seg_blur_threshold_inf = gr.Number(label='SEG blur threshold inf', value=9999.0) + guidance_seg_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='SEG start', value=0.0) + guidance_seg_stop = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='SEG stop', value=1.0) + guidance_seg_layers = gr.Textbox(label='SEG layers', value='7, 8, 9', placeholder='Comma-separated layer indices, e.g. 7,8,9') + guidance_seg_config = gr.Dropdown(choices=[None, 'config1', 'config2'], value=None, label='SEG config') + guidance_seg_args = [guidance_seg_scale, guidance_seg_blur_sigma, guidance_seg_blur_threshold_inf, guidance_seg_start, guidance_seg_stop, guidance_seg_layers, guidance_seg_config] + + tcfg_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with tcfg_group: + pass + + fdg_group = gr.Accordion(open=True, label='Advanced guidance params', elem_classes=["small-accordion"], visible=False) + with fdg_group: + guidance_fdg_scales = gr.Textbox(label='FDG scales', value='10.0, 5.0', placeholder='Comma-separated scales, e.g. 10.0,5.0') + guidance_fdg_weights = gr.Textbox(label='FDG weights', value='1.0', placeholder='Single float or comma-separated weights, e.g. 1.0 or 1.0,0.5') + guidance_fdg_rescale_space = gr.Dropdown(choices=['data', 'freq'], value='data', label='FDG rescale space') + guidance_fdg_args = [guidance_fdg_scales, guidance_fdg_weights, guidance_fdg_rescale_space] + + def adv_visibility(guidance_name): + return [ + gr.update(visible=guidance_name.startswith('Auto')), + gr.update(visible=guidance_name.startswith('Zero')), + gr.update(visible=guidance_name.startswith('PAG')), + gr.update(visible=guidance_name.startswith('APG')), + gr.update(visible=guidance_name.startswith('SLG')), + gr.update(visible=guidance_name.startswith('SEG')), + gr.update(visible=guidance_name.startswith('TCFG')), + gr.update(visible=guidance_name.startswith('FDG')), + ] + guidance_name.change(fn=adv_visibility, inputs=[guidance_name], outputs=[auto_group, zero_group, pag_group, apg_group, slg_group, seg_group, tcfg_group, fdg_group]) + + gr.HTML(value='

Fallback guidance

', visible=shared.opts.model_modular_enable, elem_id=f"{tab}_guidance_note") + with gr.Row(elem_id=f"{tab}_cfg_row", elem_classes=['flexbox']): + cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Guidance scale', value=6.0, elem_id=f"{tab}_cfg_scale") + cfg_end = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='Guidance end', value=1.0, elem_id=f"{tab}_cfg_end") + with gr.Row(): + image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Refine guidance', value=6.0, elem_id=f"{tab}_image_cfg_scale") + diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Rescale guidance', value=0.0, elem_id=f"{tab}_image_cfg_rescale") + with gr.Row(): + diffusers_pag_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.05, label='Attention guidance', value=0.0, elem_id=f"{tab}_pag_scale") + diffusers_pag_adaptive = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Adaptive scaling', value=0.5, elem_id=f"{tab}_pag_adaptive") + + _modular_args = guidance_args + lsc_args + guidance_auto_args + guidance_zero_args + guidance_pag_args + guidance_apg_args + guidance_slg_args + guidance_seg_args + guidance_fdg_args + standard_args = [cfg_scale, image_cfg_scale, diffusers_guidance_rescale, diffusers_pag_scale, diffusers_pag_adaptive, cfg_end] + return guidance_args + standard_args diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index c1597321d..cf1324484 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -1,11 +1,11 @@ -import os -from PIL import Image import gradio as gr -from modules.call_queue import wrap_gradio_gpu_call, wrap_queued_call -from modules import timer, shared, ui_common, ui_sections, generation_parameters_copypaste, processing_vae +from modules import timer, shared, call_queue, generation_parameters_copypaste, processing_vae +from modules import ui_common, ui_sections, ui_guidance def process_interrogate(mode, ii_input_files, ii_input_dir, ii_output_dir, *ii_singles): + import os + from PIL import Image from modules.interrogate.interrogate import interrogate mode = int(mode) if mode in {0, 1, 3, 4}: @@ -33,7 +33,7 @@ def process_interrogate(mode, ii_input_files, ii_input_dir, ii_output_dir, *ii_s def create_ui(): - shared.log.debug('UI initialize: img2img') + shared.log.debug('UI initialize: tab=img2img') import modules.img2img # pylint: disable=redefined-outer-name modules.scripts_manager.scripts_current = modules.scripts_manager.scripts_img2img modules.scripts_manager.scripts_img2img.initialize_scripts(is_img2img=True, is_control=False) @@ -132,9 +132,10 @@ def create_ui(): denoising_strength = gr.Slider(minimum=0.00, maximum=0.99, step=0.01, label='Denoising strength', value=0.30, elem_id="img2img_denoising_strength") refiner_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.0, elem_id="img2img_refiner_start") - vae_type, tiling, hidiffusion, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_sections.create_advanced_inputs('img2img') + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_guidance.create_guidance_inputs('img2img') + vae_type, tiling, hidiffusion, clip_skip = ui_sections.create_advanced_inputs('img2img') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('img2img') - enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, hr_refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') + enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, hr_refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('img2img') detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution = shared.yolo.ui('img2img') # with gr.Group(elem_id="inpaint_controls", visible=False) as inpaint_controls: @@ -176,8 +177,8 @@ def create_ui(): vae_type, tiling, hidiffusion, detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, batch_count, batch_size, - cfg_scale, image_cfg_scale, - diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, + cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, refiner_start, clip_skip, denoising_strength, @@ -193,7 +194,7 @@ def create_ui(): override_settings, ] img2img_dict = dict( - fn=wrap_gradio_gpu_call(modules.img2img.img2img, extra_outputs=[None, '', ''], name='Image'), + fn=call_queue.wrap_gradio_gpu_call(modules.img2img.img2img, extra_outputs=[None, '', ''], name='Image'), _js="submit_img2img", inputs= img2img_args + img2img_script_inputs, outputs=[ @@ -228,8 +229,8 @@ def create_ui(): ) interrogate_btn.click(fn=lambda *args: process_interrogate(*args), **interrogate_args) - img2img_token_button.click(fn=wrap_queued_call(ui_common.update_token_counter), inputs=[img2img_prompt], outputs=[img2img_token_counter], show_progress = False) - img2img_negative_token_button.click(fn=wrap_queued_call(ui_common.update_token_counter), inputs=[img2img_negative_prompt], outputs=[img2img_negative_token_counter], show_progress = False) + img2img_token_button.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[img2img_prompt], outputs=[img2img_token_counter], show_progress = False) + img2img_negative_token_button.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[img2img_negative_prompt], outputs=[img2img_negative_token_counter], show_progress = False) ui_extra_networks.setup_ui(extra_networks_ui_img2img, img2img_gallery) img2img_paste_fields = [ @@ -253,8 +254,13 @@ def create_ui(): (seed, "Seed"), (subseed, "Variation seed"), (subseed_strength, "Variation strength"), + # guidance + (guidance_name, "Guidance"), + (guidance_scale, "Guidance scale"), + (guidance_rescale, "Guidance rescale"), + (guidance_start, "Guidance start"), + (guidance_stop, "Guidance stop"), # advanced - (cfg_scale, "Guidance scale"), (cfg_scale, "CFG scale"), (cfg_end, "CFG end"), (image_cfg_scale, "Image CFG scale"), diff --git a/modules/ui_loadsave.py b/modules/ui_loadsave.py index a8e3dce9b..3b6484e06 100644 --- a/modules/ui_loadsave.py +++ b/modules/ui_loadsave.py @@ -35,7 +35,6 @@ class UiLoadsave: saved_value = self.ui_settings.get(key, None) self.ui_defaults[key] = getattr(obj, field) if saved_value is None: - # self.ui_settings[key] = getattr(obj, field) pass elif condition and not condition(saved_value): pass @@ -126,15 +125,37 @@ class UiLoadsave: writefile(current_ui_settings, self.filename) def dump_defaults(self): - """saves default values to a file unless the file is present and there was an error loading default values at start""" if os.path.exists(self.filename): return self.write_to_file(self.ui_settings) + def iter_all(self, values): + updates = [] + for i, name in enumerate(self.component_mapping): + component = self.component_mapping[name] + choices = getattr(component, 'choices', None) + if type(choices) is list and len(choices) > 0: # fix gradio radio button choices being tuples + if type(choices[0]) is tuple: + choices = [c[0] for c in choices] + new_value = values[i] + if isinstance(new_value, int) and choices: + if new_value >= len(choices): + updates.append(None) + new_value = choices[new_value] + old_value = self.ui_settings.get(name, None) + default_value = self.ui_defaults.get(name, '') + if old_value == new_value: + updates.append(None) + elif old_value is None and (new_value == '' or new_value == []): + updates.append(None) + elif (new_value == default_value) and (old_value is None): + updates.append(None) + else: + updates.append((name, old_value, new_value, default_value)) + return updates + def iter_changes(self, values): for i, name in enumerate(self.component_mapping): - # if '__init__' in name: - # continue component = self.component_mapping[name] choices = getattr(component, 'choices', None) if type(choices) is list and len(choices) > 0: # fix gradio radio button choices being tuples @@ -201,16 +222,23 @@ class UiLoadsave: def ui_apply(self, *values): num_changed = 0 + num_unchanged = 0 current_ui_settings = self.read_from_file() - for name, old_value, new_value, default_value in self.iter_changes(values): - component = self.component_mapping[name] - errors.log.debug(f'Settings: name={name} component={component} old={old_value} default={default_value} new={new_value}') - num_changed += 1 - current_ui_settings[name] = new_value + for x in self.iter_all(values): + if x is None: + num_unchanged += 1 + else: + name, old_value, new_value, default_value = x + component = self.component_mapping[name] + errors.log.debug(f'Settings: name={name} component={component} old={old_value} default={default_value} new={new_value}') + num_changed += 1 + current_ui_settings[name] = new_value + # what = name.split('/')[-1] + # setattr(component, what, new_value) if num_changed == 0: return "No changes" self.write_to_file(current_ui_settings) - errors.log.info(f'UI defaults saved: {self.filename}') + errors.log.info(f'UI defaults saved: {self.filename} changes={num_changed} unchanged={num_unchanged}') return f"Wrote {num_changed} changes" def ui_submenu_apply(self, items): diff --git a/modules/ui_models.py b/modules/ui_models.py index bfb876372..34563e891 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -12,6 +12,7 @@ extra_ui = [] def create_ui(): + log.debug('UI initialize: tab=models') dummy_component = gr.Label(visible=False) with gr.Row(elem_id="models_tab"): with gr.Column(elem_id='models_output_container', scale=1): diff --git a/modules/ui_postprocessing.py b/modules/ui_postprocessing.py index b575fa67a..5ed90446a 100644 --- a/modules/ui_postprocessing.py +++ b/modules/ui_postprocessing.py @@ -16,6 +16,7 @@ def submit_process(tab_index, extras_image, image_batch, extras_batch_input_dir, def create_ui(): + shared.log.debug('UI initialize: tab=process') tab_index = gr.State(value=0) # pylint: disable=abstract-class-instantiated with gr.Row(equal_height=False, variant='compact', elem_classes="extras", elem_id="extras_tab"): with gr.Column(variant='compact'): diff --git a/modules/ui_sections.py b/modules/ui_sections.py index 07ce95b3c..683681cd5 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -146,26 +146,9 @@ def create_video_inputs(tab:str, show_always:bool=False): return video_type, video_duration, video_loop, video_pad, video_interpolate -def create_cfg_inputs(tab): - with gr.Row(elem_id=f"{tab}_cfg_row", elem_classes=['flexbox']): - cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Guidance scale', value=6.0, elem_id=f"{tab}_cfg_scale") - cfg_end = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='Guidance end', value=1.0, elem_id=f"{tab}_cfg_end") - return cfg_scale, cfg_end - - -def create_advanced_inputs(tab, base=True): +def create_advanced_inputs(tab): with gr.Accordion(open=False, label="Advanced", elem_id=f"{tab}_advanced", elem_classes=["small-accordion"]): with gr.Group(): - if base: - cfg_scale, cfg_end = create_cfg_inputs(tab) - else: - cfg_scale, cfg_end = None, None - with gr.Row(): - image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Refine guidance', value=6.0, elem_id=f"{tab}_image_cfg_scale") - diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Rescale guidance', value=0.0, elem_id=f"{tab}_image_cfg_rescale") - with gr.Row(): - diffusers_pag_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.05, label='Attention guidance', value=0.0, elem_id=f"{tab}_pag_scale") - diffusers_pag_adaptive = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Adaptive scaling', value=0.5, elem_id=f"{tab}_pag_adaptive") with gr.Row(): clip_skip = gr.Slider(label='CLiP skip', value=1, minimum=0, maximum=12, step=0.1, elem_id=f"{tab}_clip_skip", interactive=shared.opts.clip_skip_enabled) with gr.Row(elem_id=f"{tab}_vae_options"): @@ -173,7 +156,7 @@ def create_advanced_inputs(tab, base=True): with gr.Row(elem_id=f"{tab}_advanced_options"): tiling = gr.Checkbox(label='Texture tiling', value=False, elem_id=f"{tab}_tiling") hidiffusion = gr.Checkbox(label='HiDiffusion', value=False, elem_id=f"{tab}_hidiffusion") - return vae_type, tiling, hidiffusion, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, diffusers_pag_scale, diffusers_pag_adaptive, cfg_end + return vae_type, tiling, hidiffusion, clip_skip def create_correction_inputs(tab): diff --git a/modules/ui_settings.py b/modules/ui_settings.py index 9b24464cc..4cee32959 100644 --- a/modules/ui_settings.py +++ b/modules/ui_settings.py @@ -172,6 +172,7 @@ def run_settings_single(value, key, progress=False): def create_ui(): + shared.log.debug('UI initialize: tab=settings') global text_settings # pylint: disable=global-statement text_settings = gr.Textbox(elem_id="settings_json", elem_classes=["settings_json"], value=lambda: shared.opts.dumpjson(), visible=False) with gr.Row(elem_id="system_row"): diff --git a/modules/ui_txt2img.py b/modules/ui_txt2img.py index fe9a19c50..6eccbdd82 100644 --- a/modules/ui_txt2img.py +++ b/modules/ui_txt2img.py @@ -1,11 +1,10 @@ import gradio as gr -from modules.call_queue import wrap_gradio_gpu_call, wrap_queued_call -from modules import timer, shared, ui_common, ui_sections, generation_parameters_copypaste, processing_vae, images -from modules.ui_components import ToolButton # pylint: disable=unused-import +from modules import timer, shared, call_queue, generation_parameters_copypaste, processing_vae, images +from modules import ui_common, ui_sections, ui_guidance def create_ui(): - shared.log.debug('UI initialize: txt2img') + shared.log.debug('UI initialize: tab=txt2img') import modules.txt2img # pylint: disable=redefined-outer-name modules.scripts_manager.scripts_current = modules.scripts_manager.scripts_txt2img modules.scripts_manager.scripts_txt2img.initialize_scripts(is_img2img=False, is_control=False) @@ -27,14 +26,14 @@ def create_ui(): width, height = ui_sections.create_resolution_inputs('txt2img') batch_count, batch_size = ui_sections.create_batch_inputs('txt2img', accordion=False) - cfg_scale, cfg_end = ui_sections.create_cfg_inputs('txt2img') steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "txt2img") with gr.Group(elem_classes="settings-accordion"): with gr.Accordion(open=False, label="Samplers", elem_classes=["small-accordion"], elem_id="txt2img_sampler_group"): ui_sections.create_sampler_options('txt2img') seed, reuse_seed, subseed, reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w = ui_sections.create_seed_inputs('txt2img') - vae_type, tiling, hidiffusion, _cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, _cfg_end = ui_sections.create_advanced_inputs('txt2img', base=False) + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_guidance.create_guidance_inputs('txt2img') + vae_type, tiling, hidiffusion, clip_skip = ui_sections.create_advanced_inputs('txt2img') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('txt2img') enable_hr, hr_sampler_index, denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution = shared.yolo.ui('txt2img') @@ -57,6 +56,7 @@ def create_ui(): vae_type, tiling, hidiffusion, detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution, batch_count, batch_size, + guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, clip_skip, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, @@ -68,7 +68,7 @@ def create_ui(): override_settings, ] txt2img_dict = dict( - fn=wrap_gradio_gpu_call(modules.txt2img.txt2img, extra_outputs=[None, '', ''], name='Text'), + fn=call_queue.wrap_gradio_gpu_call(modules.txt2img.txt2img, extra_outputs=[None, '', ''], name='Text'), _js="submit_txt2img", inputs=txt2img_args + txt2img_script_inputs, outputs=[ @@ -106,8 +106,13 @@ def create_ui(): (seed, "Seed"), (subseed, "Variation seed"), (subseed_strength, "Variation strength"), + # guidance + (guidance_name, "Guidance"), + (guidance_scale, "Guidance scale"), + (guidance_rescale, "Guidance rescale"), + (guidance_start, "Guidance start"), + (guidance_stop, "Guidance stop"), # advanced - (cfg_scale, "Guidance scale"), (cfg_scale, "CFG scale"), (cfg_end, "CFG end"), (clip_skip, "Clip skip"), @@ -156,7 +161,7 @@ def create_ui(): txt2img_bindings = generation_parameters_copypaste.ParamBinding(paste_button=txt2img_paste, tabname="txt2img", source_text_component=txt2img_prompt, source_image_component=None) generation_parameters_copypaste.register_paste_params_button(txt2img_bindings) - txt2img_token_button.click(fn=wrap_queued_call(ui_common.update_token_counter), inputs=[txt2img_prompt], outputs=[txt2img_token_counter], show_progress = False) - txt2img_negative_token_button.click(fn=wrap_queued_call(ui_common.update_token_counter), inputs=[txt2img_negative_prompt], outputs=[txt2img_negative_token_counter], show_progress = False) + txt2img_token_button.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[txt2img_prompt], outputs=[txt2img_token_counter], show_progress = False) + txt2img_negative_token_button.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[txt2img_negative_prompt], outputs=[txt2img_negative_token_counter], show_progress = False) ui_extra_networks.setup_ui(extra_networks_ui, txt2img_gallery) diff --git a/modules/ui_video.py b/modules/ui_video.py index bb9cace94..02c787c6e 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -7,7 +7,7 @@ debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None e def create_ui(): - shared.log.debug('UI initialize: video') + shared.log.debug('UI initialize: tab=video') with gr.Blocks(analytics_enabled=False) as _video_interface: prompt, styles, negative, generate_btn, _reprocess, paste, networks_button, _token_counter, _token_button, _token_counter_negative, _token_button_negative = ui_sections.create_toprow( is_img2img=False, diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index 6d92db375..e2605ae50 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -169,6 +169,13 @@ models = { te_cls=transformers.T5EncoderModel, dit_cls=diffusers.WanTransformer3DModel, dit_folder=("transformer", "transformer_2")), + Model(name='WAN 2.2 14B VACE', + url='https://huggingface.co/linoyts/Wan2.2-VACE-Fun-14B-diffusers', + repo='linoyts/Wan2.2-VACE-Fun-14B-diffusers', + repo_cls=diffusers.WanVACEPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.WanVACETransformer3DModel, + dit_folder=("transformer", "transformer_2")), Model(name='WAN 2.1 1.3B T2V', url='https://huggingface.co/Wan-AI/Wan2.1-T2V-1.3B-Diffusers', repo='Wan-AI/Wan2.1-T2V-1.3B-Diffusers', @@ -204,13 +211,13 @@ models = { repo='Wan-AI/Wan2.1-VACE-1.3B-diffusers', repo_cls=diffusers.WanVACEPipeline, te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel), + dit_cls=diffusers.WanVACETransformer3DModel), Model(name='WAN 2.1 VACE 14B', url='https://huggingface.co/Wan-AI/Wan2.1-VACE-14B-diffusers', repo='Wan-AI/Wan2.1-VACE-14B-diffusers', repo_cls=diffusers.WanVACEPipeline, te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel), + dit_cls=diffusers.WanVACETransformer3DModel), ], 'SkyReels V2': [ Model(name='None'), diff --git a/modules/video_models/video_load.py b/modules/video_models/video_load.py index 72c411166..874df0231 100644 --- a/modules/video_models/video_load.py +++ b/modules/video_models/video_load.py @@ -40,31 +40,24 @@ def load_model(selected: models_def.Model): # transformer try: + if selected.dit_folder is None: + selected.dit_folder = ['transformer'] if isinstance(selected.dit_folder, list) or isinstance(selected.dit_folder, tuple): - # wan a14b has transformer and transformer_2 - for dit_folder in selected.dit_folder: - # get a new quant arg on every loop to prevent the quant config classes getting entangled - load_args, quant_args = model_quant.get_dit_args({}, module='Model', device_map=True) - shared.log.debug(f'Video load: module=transformer repo="{selected.dit or selected.repo}" folder="{dit_folder}" cls={selected.dit_cls.__name__} quant={model_quant.get_quant_type(quant_args)}') - kwargs[dit_folder] = selected.dit_cls.from_pretrained( - pretrained_model_name_or_path=selected.dit or selected.repo, - subfolder=dit_folder, - revision=selected.dit_revision or selected.repo_revision, - cache_dir=shared.opts.hfcache_dir, - **load_args, - **quant_args - ) - else: - load_args, quant_args = model_quant.get_dit_args({}, module='Model', device_map=True) - shared.log.debug(f'Video load: module=transformer repo="{selected.dit or selected.repo}" folder="{selected.dit_folder}" cls={selected.dit_cls.__name__} quant={model_quant.get_quant_type(quant_args)}') - kwargs["transformer"] = selected.dit_cls.from_pretrained( - pretrained_model_name_or_path=selected.dit or selected.repo, - subfolder=selected.dit_folder, - revision=selected.dit_revision or selected.repo_revision, - cache_dir=shared.opts.hfcache_dir, - **load_args, - **quant_args - ) + for dit_folder in selected.dit_folder: # wan a14b has transformer and transformer_2 + if dit_folder is not None and dit_folder not in kwargs: + # get a new quant arg on every loop to prevent the quant config classes getting entangled + load_args, quant_args = model_quant.get_dit_args({}, module='Model', device_map=True) + shared.log.debug(f'Video load: module=transformer repo="{selected.dit or selected.repo}" module="{dit_folder}" folder="{dit_folder}" cls={selected.dit_cls.__name__} quant={model_quant.get_quant_type(quant_args)}') + kwargs[dit_folder] = selected.dit_cls.from_pretrained( + pretrained_model_name_or_path=selected.dit or selected.repo, + subfolder=dit_folder, + revision=selected.dit_revision or selected.repo_revision, + cache_dir=shared.opts.hfcache_dir, + **load_args, + **quant_args + ) + else: + shared.log.debug(f'Video load: module=transformer repo="{selected.dit or selected.repo}" module="{dit_folder}" folder="{dit_folder}" cls={selected.dit_cls.__name__} skip') except Exception as e: shared.log.error(f'video load: module=transformer cls={selected.dit_cls.__name__} {e}') errors.display(e, 'video') diff --git a/modules/video_models/video_overrides.py b/modules/video_models/video_overrides.py index 5ce02e20a..856da62e6 100644 --- a/modules/video_models/video_overrides.py +++ b/modules/video_models/video_overrides.py @@ -19,6 +19,15 @@ def load_override(selected: Model): # WAN if 'WAN 2.1 14B' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLWan.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir) + if 'A14B' in selected.name or '14B VACE' in selected.name: + if shared.opts.model_wan_stage == 'combined': + kwargs['boundary_ratio'] = shared.opts.model_wan_boundary + elif shared.opts.model_wan_stage == 'high noise': + kwargs['transformer_2'] = None + kwargs['boundary_ratio'] = 0.0 + elif shared.opts.model_wan_stage == 'low noise': + kwargs['boundary_ratio'] = 1.0 + kwargs['transformer'] = None debug(f'Video overrides: model="{selected.name}" kwargs={list(kwargs)}') return kwargs @@ -32,19 +41,23 @@ def set_overrides(p: processing.StableDiffusionProcessingVideo, selected: Model) if selected.name == 'Latte 1 T2V': p.task_args['enable_temporal_attentions'] = True p.task_args['video_length'] = 16 * (max(p.frames // 16, 1)) + # SkyReels + if 'SkyReelsV2DiffusionForcing' in cls: + p.task_args['overlap_history'] = 17 # LTX if cls == 'LTXImageToVideoPipeline' or cls == 'LTXConditionPipeline': p.task_args['generator'] = None if cls == 'LTXConditionPipeline': p.task_args['strength'] = p.denoising_strength + if 'LTX' in cls: + p.task_args['width'] = 32 * (p.width // 32) + p.task_args['height'] = 32 * (p.height // 32) # WAN if 'Wan' in cls: p.task_args['width'] = 16 * (p.width // 16) p.task_args['height'] = 16 * (p.height // 16) p.frames = 4 * (max(p.frames // 4, 1)) + 1 - # LTX - if 'LTX' in cls: - p.task_args['width'] = 32 * (p.width // 32) - p.task_args['height'] = 32 * (p.height // 32) - if 'SkyReelsV2DiffusionForcing' in cls: - p.task_args['overlap_history'] = 17 + # WAN VACE + if 'WanVACEPipeline' in cls: + if (getattr(p, 'init_images', None) is not None) and (len(p.init_images) > 0): + p.task_args['reference_images'] = p.init_images diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 508980bad..33d4f5d13 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -53,7 +53,10 @@ def generate(*args, **kwargs): p.do_not_save_grid = True p.do_not_save_samples = not save_frames p.outpath_samples = shared.opts.outdir_samples or shared.opts.outdir_video - if 'I2V' in model: + if 'T2V' in model: + if init_image is not None: + shared.log.warning('Video: op=T2V init image not supported') + elif 'I2V' in model: if init_image is None: return video_utils.queue_err('init image not set') p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') @@ -66,9 +69,10 @@ def generate(*args, **kwargs): p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') p.task_args['last_image'] = images.resize_image(resize_mode=2, im=last_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') shared.log.debug(f'Video: op=FLF2V init={init_image} last={last_image} resized={p.task_args["image"]}') - elif 'T2V' in model: + elif 'VACE' in model: if init_image is not None: - shared.log.warning('Video: op=T2V init image not supported') + p.task_args['reference_images'] = [images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')] + shared.log.debug(f'Video: op=VACE reference={init_image} resized={p.task_args["reference_images"]}') else: shared.log.warning(f'Video: unknown model type "{model}"') diff --git a/modules/windows_hip_ffi.py b/modules/windows_hip_ffi.py new file mode 100644 index 000000000..c612aa43d --- /dev/null +++ b/modules/windows_hip_ffi.py @@ -0,0 +1,47 @@ +import sys + +if sys.platform == "win32": + import os + import ctypes + import ctypes.wintypes + + class hipDeviceProp(ctypes.Structure): + _fields_ = [ + ('__front__', ctypes.c_byte * 396), + ('gcnArchName', ctypes.c_char * 256), + ('__rear__', ctypes.c_byte * 820) + ] + + class HIP: + def __init__(self): + ctypes.windll.kernel32.LoadLibraryA.restype = ctypes.wintypes.HMODULE + ctypes.windll.kernel32.LoadLibraryA.argtypes = [ctypes.c_char_p] + path = os.environ.get("windir", "C:\\Windows") + "\\System32\\amdhip64_6.dll" + if not os.path.isfile(path): + path = os.environ.get("windir", "C:\\Windows") + "\\System32\\amdhip64_7.dll" + assert os.path.isfile(path) + self.handle = ctypes.windll.kernel32.LoadLibraryA(path.encode('utf-8')) + ctypes.windll.kernel32.GetLastError.restype = ctypes.wintypes.DWORD + ctypes.windll.kernel32.GetLastError.argtypes = [] + assert ctypes.windll.kernel32.GetLastError() == 0 + ctypes.windll.kernel32.GetProcAddress.restype = ctypes.c_void_p + ctypes.windll.kernel32.GetProcAddress.argtypes = [ctypes.wintypes.HMODULE, ctypes.c_char_p] + self.hipGetDeviceCount = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(ctypes.c_int))( + ctypes.windll.kernel32.GetProcAddress(self.handle, b"hipGetDeviceCount")) + self.hipGetDeviceProperties = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(hipDeviceProp), ctypes.c_int)( + ctypes.windll.kernel32.GetProcAddress(self.handle, b"hipGetDeviceProperties")) + + def __del__(self): + # Hopefully this will prevent conflicts with amdhip64_7.dll from ROCm Python packages or HIP SDK + ctypes.windll.kernel32.FreeLibrary.argtypes = [ctypes.wintypes.HMODULE] + ctypes.windll.kernel32.FreeLibrary(self.handle) + + def get_device_count(self): + count = ctypes.c_int() + assert self.hipGetDeviceCount(ctypes.byref(count)) == 0 + return count.value + + def get_device_properties(self, device_id): + prop = hipDeviceProp() + assert self.hipGetDeviceProperties(ctypes.byref(prop), device_id) == 0 + return prop diff --git a/modules/zluda_installer.py b/modules/zluda_installer.py index da0ee290f..c6fb55c23 100644 --- a/modules/zluda_installer.py +++ b/modules/zluda_installer.py @@ -119,12 +119,13 @@ def link_or_copy(src: os.PathLike, dst: os.PathLike): def load(): + assert isinstance(rocm.environment, rocm.ROCmEnvironment) global core, ml, hipBLASLt_enabled, MIOpen_enabled # pylint: disable=global-statement core = Core(ctypes.windll.LoadLibrary(os.path.join(path, 'nvcuda.dll'))) ml = ZLUDALibrary(ctypes.windll.LoadLibrary(os.path.join(path, 'nvml.dll'))) is_nightly = core.get_nightly_flag() == 1 - hipBLASLt_enabled = is_nightly and os.path.exists(rocm.blaslt_tensile_libpath) and os.path.exists(os.path.join(rocm.path, "bin", "hipblaslt.dll")) and default_agent is not None - MIOpen_enabled = is_nightly and os.path.exists(os.path.join(rocm.path, "bin", "MIOpen.dll")) + hipBLASLt_enabled = is_nightly and os.path.exists(rocm.blaslt_tensile_libpath) and os.path.exists(os.path.join(rocm.environment.path, "bin", "hipblaslt.dll")) and default_agent is not None + MIOpen_enabled = is_nightly and os.path.exists(os.path.join(rocm.environment.path, "bin", "MIOpen.dll")) if hipBLASLt_enabled: if not default_agent.blaslt_supported: @@ -147,22 +148,22 @@ def load(): os.environ["ZLUDA_NVRTC_LIB"] = os.path.join([v for v in site.getsitepackages() if v.endswith("site-packages")][0], "torch", "lib", "nvrtc64_112_0.dll") for v in HIPSDK_TARGETS: - ctypes.windll.LoadLibrary(os.path.join(rocm.path, 'bin', v)) + ctypes.windll.LoadLibrary(os.path.join(rocm.environment.path, 'bin', v)) for v in DLL_MAPPING.values(): ctypes.windll.LoadLibrary(os.path.join(path, v)) if hipBLASLt_enabled: os.environ.setdefault("DISABLE_ADDMM_CUDA_LT", "0") - ctypes.windll.LoadLibrary(os.path.join(rocm.path, 'bin', 'hipblaslt.dll')) + ctypes.windll.LoadLibrary(os.path.join(rocm.environment.path, 'bin', 'hipblaslt.dll')) ctypes.windll.LoadLibrary(os.path.join(path, 'cublasLt64_11.dll')) else: os.environ["DISABLE_ADDMM_CUDA_LT"] = "1" if MIOpen_enabled: - ctypes.windll.LoadLibrary(os.path.join(rocm.path, 'bin', 'MIOpen.dll')) + ctypes.windll.LoadLibrary(os.path.join(rocm.environment.path, 'bin', 'MIOpen.dll')) ctypes.windll.LoadLibrary(os.path.join(path, 'cudnn64_9.dll')) - def conceal(): + def postinstall(): import torch torch.version.hip = rocm.version platform = sys.platform @@ -175,4 +176,4 @@ def load(): def _join_rocm_home(*paths) -> str: return os.path.join(cpp_extension.ROCM_HOME, *paths) cpp_extension._join_rocm_home = _join_rocm_home # pylint: disable=protected-access - rocm.conceal = conceal + rocm.postinstall = postinstall diff --git a/pipelines/flux/flux_legacy_loader.py b/pipelines/flux/flux_legacy_loader.py index 6b6f9d294..cb3dad3c1 100644 --- a/pipelines/flux/flux_legacy_loader.py +++ b/pipelines/flux/flux_legacy_loader.py @@ -130,7 +130,7 @@ def load_quants(kwargs, repo_id, cache_dir, allow_quant): # pylint: disable=unus shared.log.error(f'Load module: quant=Nunchaku module=transformer repo="{repo_id}" unsupported') if nunchaku_repo is not None: shared.log.debug(f'Load module: quant=Nunchaku module=transformer repo="{nunchaku_repo}" precision={nunchaku_precision} offload={shared.opts.nunchaku_offload} attention={shared.opts.nunchaku_attention}') - kwargs['transformer'] = nunchaku.NunchakuFluxTransformer2dModel.from_pretrained(nunchaku_repo, offload=shared.opts.nunchaku_offload, torch_dtype=devices.dtype) + kwargs['transformer'] = nunchaku.NunchakuFluxTransformer2dModel.from_pretrained(nunchaku_repo, offload=shared.opts.nunchaku_offload, torch_dtype=devices.dtype, cache_dir=cache_dir) kwargs['transformer'].quantization_method = 'SVDQuant' if shared.opts.nunchaku_attention: kwargs['transformer'].set_attention_impl("nunchaku-fp16") @@ -142,7 +142,7 @@ def load_quants(kwargs, repo_id, cache_dir, allow_quant): # pylint: disable=unus nunchaku_precision = nunchaku.utils.get_precision() nunchaku_repo = 'mit-han-lab/nunchaku-t5/awq-int4-flux.1-t5xxl.safetensors' shared.log.debug(f'Load module: quant=Nunchaku module=t5 repo="{nunchaku_repo}" precision={nunchaku_precision}') - kwargs['text_encoder_2'] = nunchaku.NunchakuT5EncoderModel.from_pretrained(nunchaku_repo, torch_dtype=devices.dtype) + kwargs['text_encoder_2'] = nunchaku.NunchakuT5EncoderModel.from_pretrained(nunchaku_repo, torch_dtype=devices.dtype, cache_dir=cache_dir) kwargs['text_encoder_2'].quantization_method = 'SVDQuant' if 'text_encoder_2' not in kwargs and model_quant.check_quant('TE'): load_args, quant_args = model_quant.get_dit_args(diffusers_load_config, module='TE', device_map=True) diff --git a/pipelines/flux/flux_nunchaku.py b/pipelines/flux/flux_nunchaku.py index 1aba177aa..9a737d103 100644 --- a/pipelines/flux/flux_nunchaku.py +++ b/pipelines/flux/flux_nunchaku.py @@ -6,7 +6,9 @@ def load_flux_nunchaku(repo_id): nunchaku_precision = nunchaku.utils.get_precision() nunchaku_repo = None transformer = None - if 'flux.1-dev' in repo_id.lower(): + if 'srpo' in repo_id.lower(): + pass + elif 'flux.1-dev' in repo_id.lower(): nunchaku_repo = f"nunchaku-tech/nunchaku-flux.1-dev/svdq-{nunchaku_precision}_r32-flux.1-dev.safetensors" elif 'flux.1-schnell' in repo_id.lower(): nunchaku_repo = f"nunchaku-tech/nunchaku-flux.1-schnell/svdq-{nunchaku_precision}_r32-flux.1-schnell.safetensors" @@ -24,7 +26,12 @@ def load_flux_nunchaku(repo_id): shared.log.error(f'Load module: quant=Nunchaku module=transformer repo="{repo_id}" unsupported') if nunchaku_repo is not None: shared.log.debug(f'Load module: quant=Nunchaku module=transformer repo="{nunchaku_repo}" precision={nunchaku_precision} offload={shared.opts.nunchaku_offload} attention={shared.opts.nunchaku_attention}') - transformer = nunchaku.NunchakuFluxTransformer2dModel.from_pretrained(nunchaku_repo, offload=shared.opts.nunchaku_offload, torch_dtype=devices.dtype) + transformer = nunchaku.NunchakuFluxTransformer2dModel.from_pretrained( + nunchaku_repo, + offload=shared.opts.nunchaku_offload, + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + ) transformer.quantization_method = 'SVDQuant' if shared.opts.nunchaku_attention: transformer.set_attention_impl("nunchaku-fp16") diff --git a/pipelines/model_qwen.py b/pipelines/model_qwen.py index bb65df53e..f28313603 100644 --- a/pipelines/model_qwen.py +++ b/pipelines/model_qwen.py @@ -12,7 +12,12 @@ def load_qwen(checkpoint_info, diffusers_load_config={}): load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, module='Model') shared.log.debug(f'Load model: type=Qwen model="{checkpoint_info.name}" repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') - if 'Edit' in repo_id: + if '2509' in repo_id : + cls_name = diffusers.QwenImageEditPlusPipeline + diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["qwen-image"] = diffusers.QwenImageEditPlusPipeline + diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["qwen-image"] = diffusers.QwenImageEditPlusPipeline + diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["qwen-image"] = diffusers.QwenImageEditPlusPipeline + elif 'Edit' in repo_id: cls_name = diffusers.QwenImageEditPipeline diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["qwen-image"] = diffusers.QwenImageEditPipeline diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["qwen-image"] = diffusers.QwenImageEditPipeline @@ -26,8 +31,6 @@ def load_qwen(checkpoint_info, diffusers_load_config={}): if model_quant.check_nunchaku('Model'): from pipelines.qwen.qwen_nunchaku import load_qwen_nunchaku transformer = load_qwen_nunchaku(repo_id) - # if transformer is not None: - # cls_name = nunchaku.pipeline.pipeline_qwenimage.NunchakuQwenImagePipeline # we dont need this if 'Qwen-Image-Distill-Full' in repo_id: repo_transformer = repo_id @@ -38,7 +41,13 @@ def load_qwen(checkpoint_info, diffusers_load_config={}): transformer_subfolder = "transformer" if transformer is None: - transformer = generic.load_transformer(repo_transformer, subfolder=transformer_subfolder, cls_name=diffusers.QwenImageTransformer2DModel, load_config=diffusers_load_config, modules_dtype_dict={"minimum_6bit": ["pos_embed", "time_text_embed", "img_in", "txt_in", "norm_out", "transformer_blocks.0.img_mod.1.weight"]}) + transformer = generic.load_transformer( + repo_transformer, + subfolder=transformer_subfolder, + cls_name=diffusers.QwenImageTransformer2DModel, + load_config=diffusers_load_config, + modules_dtype_dict={"minimum_8bit": ["pos_embed", "time_text_embed", "img_in", "txt_in", "norm_out", "img_mod", "transformer_blocks.0.img_mod.1.weight"]}, + ) repo_te = 'Qwen/Qwen-Image' text_encoder = generic.load_text_encoder(repo_te, cls_name=transformers.Qwen2_5_VLForConditionalGeneration, load_config=diffusers_load_config) diff --git a/pipelines/model_sana.py b/pipelines/model_sana.py index d4ec574f6..9584263a3 100644 --- a/pipelines/model_sana.py +++ b/pipelines/model_sana.py @@ -10,9 +10,8 @@ def load_quants(kwargs, repo_id, cache_dir): import nunchaku nunchaku_precision = nunchaku.utils.get_precision() nunchaku_repo = "nunchaku-tech/nunchaku-sana/svdq-int4_r32-sana1.6b.safetensors" - # https://huggingface.co/nunchaku-tech/nunchaku-sana/blob/main/svdq-int4_r32-sana1.6b.safetensors shared.log.debug(f'Load module: quant=Nunchaku module=transformer repo="{nunchaku_repo}" precision={nunchaku_precision} attention={shared.opts.nunchaku_attention}') - kwargs['transformer'] = nunchaku.NunchakuSanaTransformer2DModel.from_pretrained(nunchaku_repo, torch_dtype=devices.dtype) + kwargs['transformer'] = nunchaku.NunchakuSanaTransformer2DModel.from_pretrained(nunchaku_repo, torch_dtype=devices.dtype, cache_dir=cache_dir) elif model_quant.check_quant('Model'): load_args, quant_args = model_quant.get_dit_args(kwargs_copy, module='Model') kwargs['transformer'] = diffusers.SanaTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, **load_args, **quant_args) diff --git a/pipelines/model_wanai.py b/pipelines/model_wanai.py index b0a7d8d85..6a9f844d5 100644 --- a/pipelines/model_wanai.py +++ b/pipelines/model_wanai.py @@ -8,6 +8,11 @@ def load_transformer(repo_id, diffusers_load_config={}, subfolder='transformer') load_args, quant_args = model_quant.get_dit_args(diffusers_load_config, module='Model', device_map=True) fn = None + if 'VACE' in repo_id: + transformer_cls = diffusers.WanVACETransformer3DModel + else: + transformer_cls = diffusers.WanTransformer3DModel + if shared.opts.sd_unet is not None and shared.opts.sd_unet != 'Default': from modules import sd_unet if shared.opts.sd_unet not in list(sd_unet.unet_dict): @@ -20,7 +25,7 @@ def load_transformer(repo_id, diffusers_load_config={}, subfolder='transformer') transformer = None elif fn is not None and 'safetensors' in fn.lower(): shared.log.debug(f'Load model: type=WanAI {subfolder}="{fn}" quant="{model_quant.get_quant(repo_id)}" args={load_args}') - transformer = diffusers.WanTransformer3DModel.from_single_file( + transformer = transformer_cls.from_single_file( fn, cache_dir=shared.opts.hfcache_dir, **load_args, @@ -28,7 +33,7 @@ def load_transformer(repo_id, diffusers_load_config={}, subfolder='transformer') ) else: shared.log.debug(f'Load model: type=WanAI {subfolder}="{repo_id}" quant="{model_quant.get_quant_type(quant_args)}" args={load_args}') - transformer = diffusers.WanTransformer3DModel.from_pretrained( + transformer = transformer_cls.from_pretrained( repo_id, subfolder=subfolder, cache_dir=shared.opts.hfcache_dir, @@ -60,16 +65,20 @@ def load_wan(checkpoint_info, diffusers_load_config={}): repo_id = sd_models.path_to_repo(checkpoint_info) sd_models.hf_auth_check(checkpoint_info) - if 'a14b' in repo_id.lower(): + boundary_ratio = None + if 'a14b' in repo_id.lower() or 'fun-14b' in repo_id.lower(): if shared.opts.model_wan_stage == 'high noise' or shared.opts.model_wan_stage == 'first': transformer = load_transformer(repo_id, diffusers_load_config, 'transformer') transformer_2 = None + boundary_ratio = 0.0 elif shared.opts.model_wan_stage == 'low noise' or shared.opts.model_wan_stage == 'second': - transformer = load_transformer(repo_id, diffusers_load_config, 'transformer_2') - transformer_2 = None + transformer = None + transformer_2 = load_transformer(repo_id, diffusers_load_config, 'transformer_2') + boundary_ratio = 1.0 elif shared.opts.model_wan_stage == 'combined' or shared.opts.model_wan_stage == 'both': transformer = load_transformer(repo_id, diffusers_load_config, 'transformer') transformer_2 = load_transformer(repo_id, diffusers_load_config, 'transformer_2') + boundary_ratio = shared.opts.model_wan_boundary else: shared.log.error(f'Load model: type=WanAI stage="{shared.opts.model_wan_stage}" unsupported') return None @@ -80,16 +89,22 @@ def load_wan(checkpoint_info, diffusers_load_config={}): text_encoder = load_text_encoder(repo_id, diffusers_load_config) load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, module='Model') - boundary_ratio = shared.opts.model_wan_boundary if transformer_2 is not None else None if 'Wan2.2-I2V' in repo_id: - cls = diffusers.WanImageToVideoPipeline + pipe_cls = diffusers.WanImageToVideoPipeline diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["wanai"] = diffusers.WanImageToVideoPipeline + elif 'Wan2.2-VACE' in repo_id: + pipe_cls = diffusers.WanVACEPipeline + diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["wanai"] = diffusers.WanVACEPipeline + diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["wanai"] = diffusers.WanVACEPipeline + diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["wanai"] = diffusers.WanVACEPipeline else: - cls = diffusers.WanPipeline + from pipelines.wan.wan_image import WanImagePipeline + pipe_cls = diffusers.WanPipeline diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["wanai"] = diffusers.WanPipeline - shared.log.debug(f'Load model: type=WanAI model="{checkpoint_info.name}" repo="{repo_id}" cls={cls.__name__} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args} stage="{shared.opts.model_wan_stage}" boundary={boundary_ratio}') - pipe = cls.from_pretrained( + diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["wanai"] = WanImagePipeline + shared.log.debug(f'Load model: type=WanAI model="{checkpoint_info.name}" repo="{repo_id}" cls={pipe_cls.__name__} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args} stage="{shared.opts.model_wan_stage}" boundary={boundary_ratio}') + pipe = pipe_cls.from_pretrained( repo_id, transformer=transformer, transformer_2=transformer_2, diff --git a/pipelines/qwen/qwen_nunchaku.py b/pipelines/qwen/qwen_nunchaku.py index 084b24a1e..7c689c7a3 100644 --- a/pipelines/qwen/qwen_nunchaku.py +++ b/pipelines/qwen/qwen_nunchaku.py @@ -15,6 +15,8 @@ def load_qwen_nunchaku(repo_id): nunchaku_repo = f"nunchaku-tech/nunchaku-qwen-image/svdq-{nunchaku_precision}_r128-qwen-image.safetensors" # r32 vs r128 elif repo_id.lower().endswith('qwen-lightning'): nunchaku_repo = f"nunchaku-tech/nunchaku-qwen-image/svdq-{nunchaku_precision}_r128-qwen-image-lightningv1.1-8steps.safetensors" # 8-step variant + elif repo_id.lower().endswith('qwen-image-edit-2509'): + nunchaku_repo = f"nunchaku-tech/nunchaku-qwen-image-edit-2509/svdq-{nunchaku_precision}_r128-qwen-image-edit-2509.safetensors" # 8-step variant elif repo_id.lower().endswith('qwen-image-edit'): nunchaku_repo = f"nunchaku-tech/nunchaku-qwen-image-edit/svdq-{nunchaku_precision}_r128-qwen-image-edit.safetensors" # 8-step variant elif repo_id.lower().endswith('qwen-lightning-edit'): @@ -23,6 +25,11 @@ def load_qwen_nunchaku(repo_id): shared.log.error(f'Load module: quant=Nunchaku module=transformer repo="{repo_id}" unsupported') if nunchaku_repo is not None: shared.log.debug(f'Load module: quant=Nunchaku module=transformer repo="{nunchaku_repo}" precision={nunchaku_precision} offload={shared.opts.nunchaku_offload} attention={shared.opts.nunchaku_attention}') - transformer = NunchakuQwenImageTransformer2DModel.from_pretrained(nunchaku_repo, offload=shared.opts.nunchaku_offload, torch_dtype=devices.dtype) # pylint: disable=no-member + transformer = NunchakuQwenImageTransformer2DModel.from_pretrained( + nunchaku_repo, + offload=shared.opts.nunchaku_offload, + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + ) # pylint: disable=no-member transformer.quantization_method = 'SVDQuant' return transformer diff --git a/pipelines/wan/wan_image.py b/pipelines/wan/wan_image.py new file mode 100644 index 000000000..bd9923e5b --- /dev/null +++ b/pipelines/wan/wan_image.py @@ -0,0 +1,119 @@ +from typing import Any, Callable, Dict, List, Optional, Union +import torch +import diffusers +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import PipelineImageInput + +from modules import devices + + +class WanImagePipeline(diffusers.WanPipeline): + def __call__( + self, + prompt: Union[str, List[str]] = None, + negative_prompt: Union[str, List[str]] = None, + height: int = 480, + width: int = 832, + num_frames: int = 81, + num_inference_steps: int = 50, + guidance_scale: float = 5.0, + guidance_scale_2: Optional[float] = None, + num_videos_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + output_type: Optional[str] = "np", + return_dict: bool = True, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + strength: float = 0.3, # new + image: PipelineImageInput = None, # new + ): + # get img2img timesteps + self.scheduler.set_timesteps(num_inference_steps, device=devices.device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength) + # monkey patch original pipeline + self.scheduler.timesteps = timesteps + # self.scheduler._step_index = 0 + self.scheduler.orig_set_timesteps = self.scheduler.set_timesteps + self.scheduler.set_timesteps = lambda *args, **kwargs: None + + # prepare latents + latents = self.img2img_prepare_latents( + image=image, + timesteps=timesteps, + dtype=devices.dtype, + device=devices.device, + generator=generator, + ) + + # call original pipeline + result = super().__call__( # pylint: disable=no-member + prompt=prompt, + negative_prompt=negative_prompt, + height=height, + width=width, + num_frames=num_frames, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + guidance_scale_2=guidance_scale_2, + num_videos_per_prompt=num_videos_per_prompt, + generator=generator, + latents=latents, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + output_type=output_type, + return_dict=return_dict, + attention_kwargs=attention_kwargs, + callback_on_step_end=callback_on_step_end, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + max_sequence_length=max_sequence_length, + ) + + # un-monkey patch original pipeline + self.scheduler.set_timesteps = self.scheduler.orig_set_timesteps + return result + + def get_timesteps(self, num_inference_steps, strength): + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + # self.scheduler.set_begin_index(t_start * self.scheduler.order) + self.scheduler.set_begin_index(0) + return timesteps, num_inference_steps - t_start + + def img2img_prepare_latents( + self, + image: torch.Tensor = None, + timesteps: torch.Tensor = None, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> torch.Tensor: + from diffusers.utils.torch_utils import randn_tensor + from diffusers.video_processor import VideoProcessor + + if isinstance(image, list): + image = image[0] # ignore batch for now + + video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) + image_tensor = video_processor.preprocess(image, None, None) # convert PIL to [B, C, H, W] # channels may need rearrange + image_tensor = image_tensor.squeeze(0).to(device=device, dtype=dtype) + image_tensor = image_tensor[None, :, None, :, :] # expand before encode to [B, C, N, H, W] + encoder_output = self.vae.encode(image_tensor) + # init_latents = encoder_output.latent_dist.mode() # argmax or sample? + init_latents = encoder_output.latent_dist.sample(generator) + + latents_mean = torch.tensor(self.vae.config.latents_mean, device=device, dtype=torch.float32).view(1, self.vae.config.z_dim, 1, 1, 1) + latents_std = 1.0 / torch.tensor(self.vae.config.latents_std, device=device, dtype=torch.float32).view(1, self.vae.config.z_dim, 1, 1, 1) + init_latents = ((init_latents.float() - latents_mean) * latents_std).to(dtype) # normalized to standard distribution range + + init_noise = randn_tensor(init_latents.shape, generator=generator, device=device, dtype=dtype) + init_timestep = timesteps[:1] + noised_latents = self.scheduler.add_noise(init_latents, init_noise, init_timestep) + + return noised_latents diff --git a/requirements.txt b/requirements.txt index 25fb3c837..6ca4153e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,7 +44,7 @@ tqdm==4.67.1 accelerate==1.10.1 opencv-contrib-python-headless==4.11.0.86 einops==0.8.1 -huggingface_hub==0.34.4 +huggingface_hub==0.35.3 numexpr==2.11.0 numpy==2.1.2 pandas==2.3.1 @@ -69,7 +69,3 @@ seam-carving ruff pylint pre-commit - -# block -torch!=2.5.0 -torchvision!=0.20.0 diff --git a/scripts/daam/__init__.py b/scripts/daam/__init__.py new file mode 100644 index 000000000..a407cfd4d --- /dev/null +++ b/scripts/daam/__init__.py @@ -0,0 +1,5 @@ +from .experiment import * +from .heatmap import * +from .hook import * +from .utils import * +from .trace import * diff --git a/scripts/daam/evaluate.py b/scripts/daam/evaluate.py new file mode 100644 index 000000000..ec9e1567b --- /dev/null +++ b/scripts/daam/evaluate.py @@ -0,0 +1,122 @@ +from collections import defaultdict +from typing import List, Union + +from scipy.optimize import linear_sum_assignment +import PIL.Image as Image +import numpy as np +import torch +import torch.nn.functional as F + + +__all__ = ['compute_iou', 'MeanEvaluator', 'load_mask', 'compute_ioa'] + + +def compute_iou(a: torch.Tensor, b: torch.Tensor) -> float: + if a.shape[0] != b.shape[0]: + a = F.interpolate(a.unsqueeze(0).unsqueeze(0).float(), size=b.shape, mode='bicubic').squeeze() + a[a < 1] = 0 + a[a >= 1] = 1 + + intersection = (a * b).sum() + union = a.sum() + b.sum() - intersection + + return (intersection / (union + 1e-8)).item() + + +def compute_ioa(a: torch.Tensor, b: torch.Tensor) -> float: + if a.shape[0] != b.shape[0]: + a = F.interpolate(a.unsqueeze(0).unsqueeze(0).float(), size=b.shape, mode='bicubic').squeeze() + a[a < 1] = 0 + a[a >= 1] = 1 + + intersection = (a * b).sum() + area = a.sum() + + return (intersection / (area + 1e-8)).item() + + +def load_mask(path: str) -> torch.Tensor: + mask = np.array(Image.open(path)) + mask = torch.from_numpy(mask).float()[:, :, 3] # use alpha channel + mask = (mask > 0).float() + + return mask + + +class UnsupervisedEvaluator: + def __init__(self, name: str = 'UnsupervisedEvaluator'): + self.name = name + self.ious = defaultdict(list) + self.num_samples = 0 + + def log_iou(self, preds: Union[torch.Tensor, List[torch.Tensor]], truth: torch.Tensor, gt_idx: int = 0, pred_idx: int = 0): + if not isinstance(preds, list): + preds = [preds] + + iou = max(compute_iou(pred, truth) for pred in preds) + self.ious[gt_idx].append((pred_idx, iou)) + + @property + def mean_iou(self) -> float: + n = max(max(self.ious), max([y[0] for x in self.ious.values() for y in x])) + 1 + iou_matrix = np.zeros((n, n)) + count_matrix = np.zeros((n, n)) + + for gt_idx, ious in self.ious.items(): + for pred_idx, iou in ious: + iou_matrix[gt_idx, pred_idx] += iou + count_matrix[gt_idx, pred_idx] += 1 + + row_ind, col_ind = linear_sum_assignment(iou_matrix, maximize=True) + return iou_matrix[row_ind, col_ind].sum() / count_matrix[row_ind, col_ind].sum() + + def increment(self): + self.num_samples += 1 + + def __len__(self) -> int: + return self.num_samples + + def __str__(self): + return f'{self.name}<{self.mean_iou:.4f} (mIoU) {len(self)} samples>' + + +class MeanEvaluator: + def __init__(self, name: str = 'MeanEvaluator'): + self.ious: List[float] = [] + self.intensities: List[float] = [] + self.name = name + + def log_iou(self, preds: Union[torch.Tensor, List[torch.Tensor]], truth: torch.Tensor): + if not isinstance(preds, list): + preds = [preds] + + self.ious.append(max(compute_iou(pred, truth) for pred in preds)) + return self + + def log_intensity(self, pred: torch.Tensor): + self.intensities.append(pred.mean().item()) + return self + + @property + def mean_iou(self) -> float: + return np.mean(self.ious) + + @property + def mean_intensity(self) -> float: + return np.mean(self.intensities) + + @property + def ci95_miou(self) -> float: + return 1.96 * np.std(self.ious) / np.sqrt(len(self.ious)) + + def __len__(self) -> int: + return max(len(self.ious), len(self.intensities)) + + def __str__(self): + return f'{self.name}<{self.mean_iou:.4f} (±{self.ci95_miou:.3f} mIoU) {self.mean_intensity:.4f} (mInt) {len(self)} samples>' + + +if __name__ == '__main__': + mask = load_mask('truth/output/452/sink.gt.png') + + print(MeanEvaluator().log_iou(mask, mask)) diff --git a/scripts/daam/experiment.py b/scripts/daam/experiment.py new file mode 100644 index 000000000..4465a3054 --- /dev/null +++ b/scripts/daam/experiment.py @@ -0,0 +1,344 @@ +from pathlib import Path +from typing import List, Optional, Dict, Any, Union +from dataclasses import dataclass +import json + +from transformers import PreTrainedTokenizer, AutoTokenizer +import PIL.Image +import numpy as np +import torch + +from .utils import auto_autocast +from .evaluate import load_mask + + +__all__ = ['GenerationExperiment', 'COCO80_LABELS', 'COCOSTUFF27_LABELS', 'COCO80_INDICES', 'build_word_list_coco80'] + + +COCO80_LABELS: List[str] = [ + 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', + 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', + 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', + 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', + 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', + 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', + 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', + 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', + 'hair drier', 'toothbrush' +] + +COCO80_INDICES: Dict[str, int] = {x: i for i, x in enumerate(COCO80_LABELS)} + +UNUSED_LABELS: List[str] = [f'__unused_{i}__' for i in range(1, 200)] + +COCOSTUFF27_LABELS: List[str] = [ + 'electronic', 'appliance', 'food', 'furniture', 'indoor', 'kitchen', 'accessory', 'animal', 'outdoor', 'person', + 'sports', 'vehicle', 'ceiling', 'floor', 'food', 'furniture', 'rawmaterial', 'textile', 'wall', 'window', + 'building', 'ground', 'plant', 'sky', 'solid', 'structural', 'water' +] + +COCO80_ONTOLOGY = { + 'two-wheeled vehicle': ['bicycle', 'motorcycle'], + 'vehicle': ['two-wheeled vehicle', 'four-wheeled vehicle'], + 'four-wheeled vehicle': ['bus', 'truck', 'car'], + 'four-legged animals': ['livestock', 'pets', 'wild animals'], + 'livestock': ['cow', 'horse', 'sheep'], + 'pets': ['cat', 'dog'], + 'wild animals': ['elephant', 'bear', 'zebra', 'giraffe'], + 'bags': ['backpack', 'handbag', 'suitcase'], + 'sports boards': ['snowboard', 'surfboard', 'skateboard'], + 'utensils': ['fork', 'knife', 'spoon'], + 'receptacles': ['bowl', 'cup'], + 'fruits': ['banana', 'apple', 'orange'], + 'foods': ['fruits', 'meals', 'desserts'], + 'meals': ['sandwich', 'hot dog', 'pizza'], + 'desserts': ['cake', 'donut'], + 'furniture': ['chair', 'couch', 'bench'], + 'electronics': ['monitors', 'appliances'], + 'monitors': ['tv', 'cell phone', 'laptop'], + 'appliances': ['oven', 'toaster', 'refrigerator'] +} + +COCO80_TO_27 = { + 'bicycle': 'vehicle', 'car': 'vehicle', 'motorcycle': 'vehicle', 'airplane': 'vehicle', 'bus': 'vehicle', + 'train': 'vehicle', 'truck': 'vehicle', 'boat': 'vehicle', 'traffic light': 'accessory', 'fire hydrant': 'accessory', + 'stop sign': 'accessory', 'parking meter': 'accessory', 'bench': 'furniture', 'bird': 'animal', 'cat': 'animal', + 'dog': 'animal', 'horse': 'animal', 'sheep': 'animal', 'cow': 'animal', 'elephant': 'animal', 'bear': 'animal', + 'zebra': 'animal', 'giraffe': 'animal', 'backpack': 'accessory', 'umbrella': 'accessory', 'handbag': 'accessory', + 'tie': 'accessory', 'suitcase': 'accessory', 'frisbee': 'sports', 'skis': 'sports', 'snowboard': 'sports', + 'sports ball': 'sports', 'kite': 'sports', 'baseball bat': 'sports', 'baseball glove': 'sports', + 'skateboard': 'sports', 'surfboard': 'sports', 'tennis racket': 'sports', 'bottle': 'food', 'wine glass': 'food', + 'cup': 'food', 'fork': 'food', 'knife': 'food', 'spoon': 'food', 'bowl': 'food', 'banana': 'food', 'apple': 'food', + 'sandwich': 'food', 'orange': 'food', 'broccoli': 'food', 'carrot': 'food', 'hot dog': 'food', 'pizza': 'food', + 'donut': 'food', 'cake': 'food', 'chair': 'furniture', 'couch': 'furniture', 'potted plant': 'plant', + 'bed': 'furniture', 'dining table': 'furniture', 'toilet': 'furniture', 'tv': 'electronic', 'laptop': 'electronic', + 'mouse': 'electronic', 'remote': 'electronic', 'keyboard': 'electronic', 'cell phone': 'electronic', + 'microwave': 'appliance', 'oven': 'appliance', 'toaster': 'appliance', 'sink': 'appliance', + 'refrigerator': 'appliance', 'book': 'indoor', 'clock': 'indoor', 'vase': 'indoor', 'scissors': 'indoor', + 'teddy bear': 'indoor', 'hair drier': 'indoor', 'toothbrush': 'indoor' +} + + +def build_word_list_coco80() -> Dict[str, List[str]]: + words_map = COCO80_ONTOLOGY.copy() + words_map = {k: v for k, v in words_map.items() if not any(item in COCO80_ONTOLOGY for item in v)} + + return words_map + + +def _add_mask(masks: Dict[str, torch.Tensor], word: str, mask: torch.Tensor, simplify80: bool = False) -> Dict[str, torch.Tensor]: + if simplify80: + word = COCO80_TO_27.get(word, word) + + if word in masks: + masks[word] = masks[word.lower()] + mask + masks[word].clamp_(0, 1) + else: + masks[word] = mask + + return masks + + +@dataclass +class GenerationExperiment: + """Class to hold experiment parameters. Pickleable.""" + image: PIL.Image.Image + global_heat_map: torch.Tensor + prompt: str + + seed: int = None + id: str = '.' + path: Optional[Path] = None + + truth_masks: Optional[Dict[str, torch.Tensor]] = None + prediction_masks: Optional[Dict[str, torch.Tensor]] = None + annotations: Optional[Dict[str, Any]] = None + subtype: Optional[str] = '.' + tokenizer: AutoTokenizer = None + + def __post_init__(self): + if isinstance(self.path, str): + self.path = Path(self.path) + + self.path = None if self.path is None else self.path / self.id + + def nsfw(self) -> bool: + return np.sum(np.array(self.image)) == 0 + + def heat_map(self, tokenizer: AutoTokenizer = None): + if tokenizer is None: + tokenizer = self.tokenizer + + from daam import GlobalHeatMap + return GlobalHeatMap(tokenizer, self.prompt, self.global_heat_map) + + def clear_checkpoint(self): + path = self if isinstance(self, Path) else self.path + + (path / 'generation.pt').unlink(missing_ok=True) + + def save(self, path: str = None, heat_maps: bool = True, tokenizer: AutoTokenizer = None): + if path is None: + path = self.path + else: + path = Path(path) / self.id + + if tokenizer is None: + tokenizer = self.tokenizer + + (path / self.subtype).mkdir(parents=True, exist_ok=True) + torch.save(self, path / self.subtype / 'generation.pt') + self.image.save(path / self.subtype / 'output.png') + + with (path / 'prompt.txt').open('w') as f: + f.write(self.prompt) + + with (path / 'seed.txt').open('w') as f: + f.write(str(self.seed)) + + if self.truth_masks is not None: + for name, mask in self.truth_masks.items(): + im = PIL.Image.fromarray((mask * 255).unsqueeze(-1).expand(-1, -1, 4).byte().numpy()) + im.save(path / f'{name.lower()}.gt.png') + + if heat_maps and tokenizer is not None: + self.save_all_heat_maps(tokenizer) + + self.save_annotations() + + def save_annotations(self, path: Path = None): + if path is None: + path = self.path + + if self.annotations is not None: + with (path / 'annotations.json').open('w') as f: + json.dump(self.annotations, f) + + def _load_truth_masks(self, simplify80: bool = False) -> Dict[str, torch.Tensor]: + masks = {} + + for mask_path in self.path.glob('*.gt.png'): + word = mask_path.name.split('.gt.png')[0].lower() + mask = load_mask(str(mask_path)) + _add_mask(masks, word, mask, simplify80) + + return masks + + def _load_pred_masks(self, pred_prefix, composite=False, simplify80=False, vocab=None): + # type: (str, bool, bool, List[str] | None) -> Dict[str, torch.Tensor] + masks = {} + + if vocab is None: + vocab = UNUSED_LABELS + + if composite: + try: + im = PIL.Image.open(self.path / self.subtype / f'composite.{pred_prefix}.pred.png') + im = np.array(im) + + for mask_idx in np.unique(im): + mask = torch.from_numpy((im == mask_idx).astype(np.float32)) + _add_mask(masks, vocab[mask_idx], mask, simplify80) + except FileNotFoundError: + pass + else: + for mask_path in (self.path / self.subtype).glob(f'*.{pred_prefix}.pred.png'): + mask = load_mask(str(mask_path)) + word = mask_path.name.split(f'.{pred_prefix}.pred')[0].lower() + _add_mask(masks, word, mask, simplify80) + + return masks + + def clear_prediction_masks(self, name: str): + path = self if isinstance(self, Path) else self.path + path = path / self.subtype + + for mask_path in path.glob(f'*.{name}.pred.png'): + mask_path.unlink() + + def save_prediction_mask(self, mask: torch.Tensor, word: str, name: str): + path = self if isinstance(self, Path) else self.path + im = PIL.Image.fromarray((mask * 255).unsqueeze(-1).expand(-1, -1, 4).cpu().byte().numpy()) + im.save(path / self.subtype / f'{word.lower()}.{name}.pred.png') + + def save_heat_map( + self, + word: str, + tokenizer: PreTrainedTokenizer = None, + crop: int = None, + output_prefix: str = '', + absolute: bool = False + ) -> Path: + from .trace import GlobalHeatMap # because of cyclical import + + if tokenizer is None: + tokenizer = self.tokenizer + + with auto_autocast(dtype=torch.float32): + path = self.path / self.subtype / f'{output_prefix}{word.lower()}.heat_map.png' + heat_map = GlobalHeatMap(tokenizer, self.prompt, self.global_heat_map) + heat_map.compute_word_heat_map(word).expand_as(self.image, color_normalize=not absolute, out_file=path, plot=True) + + return path + + def save_all_heat_maps(self, tokenizer: PreTrainedTokenizer = None, crop: int = None) -> Dict[str, Path]: + path_map = {} + + if tokenizer is None: + tokenizer = self.tokenizer + + for word in self.prompt.split(' '): + try: + path = self.save_heat_map(word, tokenizer, crop=crop) + path_map[word] = path + except: + pass + + return path_map + + @staticmethod + def contains_truth_mask(path: Union[str, Path], prompt_id: str = None) -> bool: + if prompt_id is None: + return any(Path(path).glob('*.gt.png')) + else: + return any((Path(path) / prompt_id).glob('*.gt.png')) + + @staticmethod + def read_seed(path: Union[str, Path], prompt_id: str = None) -> int: + if prompt_id is None: + return int(Path(path).joinpath('seed.txt').read_text()) + else: + return int(Path(path).joinpath(prompt_id).joinpath('seed.txt').read_text()) + + @staticmethod + def has_annotations(path: Union[str, Path]) -> bool: + return Path(path).joinpath('annotations.json').exists() + + @staticmethod + def has_experiment(path: Union[str, Path], prompt_id: str) -> bool: + return (Path(path) / prompt_id / 'generation.pt').exists() + + @staticmethod + def read_prompt(path: Union[str, Path], prompt_id: str = None) -> str: + if prompt_id is None: + prompt_id = '.' + + with (Path(path) / prompt_id / 'prompt.txt').open('r') as f: + return f.read().strip() + + def _try_load_annotations(self): + if not (self.path / 'annotations.json').exists(): + return None + + return json.load((self.path / 'annotations.json').open()) + + def annotate(self, key: str, value: Any) -> 'GenerationExperiment': + if self.annotations is None: + self.annotations = {} + + self.annotations[key] = value + + return self + + @classmethod + def load( + cls, + path, + pred_prefix='daam', + composite=False, + simplify80=False, + vocab=None, + subtype='.', + all_subtypes=False + ): + # type: (str, str, bool, bool, List[str] | None, str, bool) -> GenerationExperiment | List[GenerationExperiment] + if all_subtypes: + experiments = [] + + for directory in Path(path).iterdir(): + if not directory.is_dir(): + continue + + try: + experiments.append(cls.load( + path, + pred_prefix=pred_prefix, + composite=composite, + simplify80=simplify80, + vocab=vocab, + subtype=directory.name + )) + except: + pass + + return experiments + + path = Path(path) + exp = torch.load(path / subtype / 'generation.pt') + exp.subtype = subtype + exp.path = path + exp.truth_masks = exp._load_truth_masks(simplify80=simplify80) + exp.prediction_masks = exp._load_pred_masks(pred_prefix, composite=composite, simplify80=simplify80, vocab=vocab) + exp.annotations = exp._try_load_annotations() + + return exp diff --git a/scripts/daam/heatmap.py b/scripts/daam/heatmap.py new file mode 100644 index 000000000..0f7a311f0 --- /dev/null +++ b/scripts/daam/heatmap.py @@ -0,0 +1,185 @@ +import io +from collections import defaultdict +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, Tuple, Set, Iterable + +from matplotlib import pyplot as plt +import numpy as np +import PIL.Image +import spacy.tokens +import torch +import torch.nn.functional as F + +from .evaluate import compute_ioa +from .utils import compute_token_merge_indices, cached_nlp, auto_autocast + +__all__ = ['GlobalHeatMap', 'RawHeatMapCollection', 'WordHeatMap', 'ParsedHeatMap', 'SyntacticHeatMapPair'] + + +def plot_overlay_heat_map(im, heat_map, word=None, out_file=None, crop=None, color_normalize=True, ax=None, cmap='jet'): + # type: (PIL.Image.Image | np.ndarray, torch.Tensor, str, Path, int, bool, plt.Axes) -> None + if ax is None: + plt.rcParams['font.size'] = 16 + plt.rcParams['figure.facecolor'] = 'black' + plt.rcParams['text.color'] = 'white' + plt.rcParams['axes.labelcolor'] = 'white' + plt.rcParams['xtick.color'] = 'black' + plt.rcParams['ytick.color'] = 'black' + plt.clf() + plt_ = plt + else: + plt_ = ax + + with auto_autocast(dtype=torch.float32): + im = np.array(im) + + if crop is not None: + heat_map = heat_map.squeeze()[crop:-crop, crop:-crop] + im = im[crop:-crop, crop:-crop] + + if color_normalize: + plt_.imshow(heat_map.squeeze().cpu().numpy(), cmap=cmap) + else: + heat_map = heat_map.clamp_(min=0, max=1) + plt_.imshow(heat_map.squeeze().cpu().numpy(), cmap=cmap, vmin=0.0, vmax=1.0) + + im = torch.from_numpy(im).float() / 255 + im = torch.cat((im, (1 - heat_map.unsqueeze(-1))), dim=-1) + plt_.imshow(im) + + if word is not None: + if ax is None: + plt.title(word) + else: + ax.set_title(word) + + if out_file is not None: + plt.savefig(out_file) + + buf = io.BytesIO() + plt.savefig(buf, format='png', bbox_inches='tight') + buf.seek(0) + image = PIL.Image.open(buf) + return image + + +class WordHeatMap: + def __init__(self, heatmap: torch.Tensor, word: str = None, word_idx: int = None): + self.word = word + self.word_idx = word_idx + self.heatmap = heatmap + + @property + def value(self): + return self.heatmap + + def plot_overlay(self, image, out_file=None, color_normalize=True, ax=None, cmap='jet', **expand_kwargs): + # type: (PIL.Image.Image | np.ndarray, Path, bool, plt.Axes, Dict[str, Any]) -> None + return plot_overlay_heat_map( + image, + self.expand_as(image, **expand_kwargs), + word=self.word, + out_file=out_file, + color_normalize=color_normalize, + ax=ax, + cmap=cmap, + ) + + def expand_as(self, image, absolute=False, threshold=None, plot=False, **plot_kwargs): + # type: (PIL.Image.Image, bool, float, bool, Dict[str, Any]) -> torch.Tensor + im = self.heatmap.unsqueeze(0).unsqueeze(0) + im = F.interpolate(im.float().detach(), size=(image.size[0], image.size[1]), mode='bicubic') + + if not absolute: + im = (im - im.min()) / (im.max() - im.min() + 1e-8) + + if threshold: + im = (im > threshold).float() + + im = im.cpu().detach().squeeze() + + if plot: + self.plot_overlay(image, **plot_kwargs) + + return im + + def compute_ioa(self, other: 'WordHeatMap'): + return compute_ioa(self.heatmap, other.heatmap) + + +@dataclass +class SyntacticHeatMapPair: + head_heat_map: WordHeatMap + dep_heat_map: WordHeatMap + head_text: str + dep_text: str + relation: str + + +@dataclass +class ParsedHeatMap: + word_heat_map: WordHeatMap + token: spacy.tokens.Token + + +class GlobalHeatMap: + def __init__(self, tokenizer: Any, prompt: str, heat_maps: torch.Tensor): + self.tokenizer = tokenizer + self.heat_maps = heat_maps + self.prompt = prompt + self.compute_word_heat_map = lru_cache(maxsize=50)(self.compute_word_heat_map) + + def compute_word_heat_map(self, word: str, word_idx: int = None, offset_idx: int = 0) -> WordHeatMap: + merge_idxs, word_idx = compute_token_merge_indices(self.tokenizer, self.prompt, word, word_idx, offset_idx) + return WordHeatMap(self.heat_maps[merge_idxs].mean(0), word, word_idx) + + def parsed_heat_maps(self) -> Iterable[ParsedHeatMap]: + for token in cached_nlp(self.prompt): + try: + heat_map = self.compute_word_heat_map(token.text) + yield ParsedHeatMap(heat_map, token) + except ValueError: + pass + + def dependency_relations(self) -> Iterable[SyntacticHeatMapPair]: + for token in cached_nlp(self.prompt): + if token.dep_ != 'ROOT': + try: + dep_heat_map = self.compute_word_heat_map(token.text) + head_heat_map = self.compute_word_heat_map(token.head.text) + + yield SyntacticHeatMapPair(head_heat_map, dep_heat_map, token.head.text, token.text, token.dep_) + except ValueError: + pass + + +RawHeatMapKey = Tuple[int, int, int] # factor, layer, head + + +class RawHeatMapCollection: + def __init__(self): + self.ids_to_heatmaps: Dict[RawHeatMapKey, torch.Tensor] = defaultdict(lambda: 0.0) + self.ids_to_num_maps: Dict[RawHeatMapKey, int] = defaultdict(lambda: 0) + + def update(self, factor: int, layer_idx: int, head_idx: int, heatmap: torch.Tensor): + with auto_autocast(dtype=torch.float32): + key = (factor, layer_idx, head_idx) + self.ids_to_heatmaps[key] = self.ids_to_heatmaps[key] + heatmap + + def factors(self) -> Set[int]: + return set(key[0] for key in self.ids_to_heatmaps.keys()) + + def layers(self) -> Set[int]: + return set(key[1] for key in self.ids_to_heatmaps.keys()) + + def heads(self) -> Set[int]: + return set(key[2] for key in self.ids_to_heatmaps.keys()) + + def __iter__(self): + return iter(self.ids_to_heatmaps.items()) + + def clear(self): + self.ids_to_heatmaps.clear() + self.ids_to_num_maps.clear() diff --git a/scripts/daam/hook.py b/scripts/daam/hook.py new file mode 100644 index 000000000..7c2866b63 --- /dev/null +++ b/scripts/daam/hook.py @@ -0,0 +1,127 @@ +from typing import List, Generic, TypeVar +import functools +import itertools + +from diffusers import UNet2DConditionModel +from diffusers.models.attention_processor import Attention +import torch.nn as nn + + +__all__ = ['ObjectHooker', 'ModuleLocator', 'AggregateHooker', 'UNetCrossAttentionLocator'] + + +ModuleType = TypeVar('ModuleType') +ModuleListType = TypeVar('ModuleListType', bound=List) + + +class ModuleLocator(Generic[ModuleType]): + def locate(self, model: nn.Module) -> List[ModuleType]: + raise NotImplementedError + + +class ObjectHooker(Generic[ModuleType]): + def __init__(self, module: ModuleType): + self.module: ModuleType = module + self.hooked = False + self.old_state = {} + + def __enter__(self): + self.hook() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.unhook() + + def hook(self): + if self.hooked: + raise RuntimeError('Already hooked module') + + self.old_state = {} + self.hooked = True + self._hook_impl() + + return self + + def unhook(self): + if not self.hooked: + raise RuntimeError('Module is not hooked') + + for k, v in self.old_state.items(): + if k.startswith('old_fn_'): + setattr(self.module, k[7:], v) + + self.hooked = False + self._unhook_impl() + + return self + + def monkey_patch(self, fn_name, fn, strict: bool = True): + try: + self.old_state[f'old_fn_{fn_name}'] = getattr(self.module, fn_name) + setattr(self.module, fn_name, functools.partial(fn, self.module)) + except AttributeError: + if strict: + raise + + def monkey_super(self, fn_name, *args, **kwargs): + return self.old_state[f'old_fn_{fn_name}'](*args, **kwargs) + + def _hook_impl(self): + raise NotImplementedError + + def _unhook_impl(self): + pass + + +class AggregateHooker(ObjectHooker[ModuleListType]): + def _hook_impl(self): + for h in self.module: + h.hook() + + def _unhook_impl(self): + for h in self.module: + h.unhook() + + def register_hook(self, hook: ObjectHooker): + self.module.append(hook) + + +class UNetCrossAttentionLocator(ModuleLocator[Attention]): + def __init__(self, restrict: bool = None, locate_middle_block: bool = False): + self.restrict = restrict + self.layer_names = [] + self.locate_middle_block = locate_middle_block + + def locate(self, model: UNet2DConditionModel) -> List[Attention]: + """ + Locate all cross-attention modules in a UNet2DConditionModel. + + Args: + model (`UNet2DConditionModel`): The model to locate the cross-attention modules in. + + Returns: + `List[Attention]`: The list of cross-attention modules. + """ + self.layer_names.clear() + blocks_list = [] + up_names = ['up'] * len(model.up_blocks) + down_names = ['down'] * len(model.down_blocks) + + for unet_block, name in itertools.chain( + zip(model.up_blocks, up_names), + zip(model.down_blocks, down_names), + zip([model.mid_block], ['mid']) if self.locate_middle_block else [], + ): + if 'CrossAttn' in unet_block.__class__.__name__: + blocks = [] + + for spatial_transformer in unet_block.attentions: + for transformer_block in spatial_transformer.transformer_blocks: + blocks.append(transformer_block.attn2) + + blocks = [b for idx, b in enumerate(blocks) if self.restrict is None or idx in self.restrict] + names = [f'{name}-attn-{i}' for i in range(len(blocks)) if self.restrict is None or i in self.restrict] + blocks_list.extend(blocks) + self.layer_names.extend(names) + + return blocks_list diff --git a/scripts/daam/trace.py b/scripts/daam/trace.py new file mode 100644 index 000000000..625e3d402 --- /dev/null +++ b/scripts/daam/trace.py @@ -0,0 +1,323 @@ +from pathlib import Path +from typing import List, Type, Any, Dict, Union +import math + +from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.attention_processor import Attention +import numpy as np +import PIL.Image as Image +import torch +import torch.nn.functional as F + +from .utils import cache_dir, auto_autocast +from .experiment import GenerationExperiment +from .heatmap import RawHeatMapCollection, GlobalHeatMap +from .hook import ObjectHooker, AggregateHooker, UNetCrossAttentionLocator + + +__all__ = ['trace', 'DiffusionHeatMapHooker', 'GlobalHeatMap'] + + +class DiffusionHeatMapHooker(AggregateHooker): + def __init__( + self, + pipeline: Union[StableDiffusionPipeline, StableDiffusionXLPipeline], + low_memory: bool = False, + load_heads: bool = False, + save_heads: bool = False, + data_dir: str = None + ): + self.all_heat_maps = RawHeatMapCollection() + h = (pipeline.unet.config.sample_size * pipeline.vae_scale_factor) + self.latent_hw = 4096 if h == 512 or h == 1024 else 9216 # 64x64 or 96x96 depending on if it's 2.0-v or 2.0 + locate_middle = load_heads or save_heads + self.locator = UNetCrossAttentionLocator(restrict={0} if low_memory else None, locate_middle_block=locate_middle) + self.last_prompt: str = '' + self.last_image: Image = None + self.time_idx = 0 + self._gen_idx = 0 + + modules = [ + UNetCrossAttentionHooker( + x, + self, + layer_idx=idx, + latent_hw=self.latent_hw, + load_heads=load_heads, + save_heads=save_heads, + data_dir=data_dir + ) for idx, x in enumerate(self.locator.locate(pipeline.unet)) + ] + + modules.append(PipelineHooker(pipeline, self)) + + if type(pipeline) == StableDiffusionXLPipeline: + modules.append(ImageProcessorHooker(pipeline.image_processor, self)) + + super().__init__(modules) + self.pipe = pipeline + + def time_callback(self, *args, **kwargs): + self.time_idx += 1 + + @property + def layer_names(self): + return self.locator.layer_names + + def to_experiment(self, path, seed=None, id='.', subtype='.', **compute_kwargs): + # type: (Union[Path, str], int, str, str, Dict[str, Any]) -> GenerationExperiment + """Exports the last generation call to a serializable generation experiment.""" + + return GenerationExperiment( + self.last_image, + self.compute_global_heat_map(**compute_kwargs).heat_maps, + self.last_prompt, + seed=seed, + id=id, + subtype=subtype, + path=path, + tokenizer=self.pipe.tokenizer, + ) + + def compute_global_heat_map(self, prompt=None, factors=None, head_idx=None, layer_idx=None, normalize=False): + # type: (str, List[float], int, int, bool) -> GlobalHeatMap + """ + Compute the global heat map for the given prompt, aggregating across time (inference steps) and space (different + spatial transformer block heat maps). + + Args: + prompt: The prompt to compute the heat map for. If none, uses the last prompt that was used for generation. + factors: Restrict the application to heat maps with spatial factors in this set. If `None`, use all sizes. + head_idx: Restrict the application to heat maps with this head index. If `None`, use all heads. + layer_idx: Restrict the application to heat maps with this layer index. If `None`, use all layers. + + Returns: + A heat map object for computing word-level heat maps. + """ + heat_maps = self.all_heat_maps + + if prompt is None: + prompt = self.last_prompt + + if factors is None: + factors = {0, 1, 2, 4, 8, 16, 32, 64} + else: + factors = set(factors) + + all_merges = [] + x = int(np.sqrt(self.latent_hw)) + + with auto_autocast(dtype=torch.float32): + for (factor, layer, head), heat_map in heat_maps: + if factor in factors and (head_idx is None or head_idx == head) and (layer_idx is None or layer_idx == layer): + heat_map = heat_map.unsqueeze(1) + # The clamping fixes undershoot. + all_merges.append(F.interpolate(heat_map, size=(x, x), mode='bicubic').clamp_(min=0)) + + try: + maps = torch.stack(all_merges, dim=0) + except RuntimeError: + if head_idx is not None or layer_idx is not None: + raise RuntimeError('No heat maps found for the given parameters.') + else: + raise RuntimeError('No heat maps found. Did you forget to call `with trace(...)` during generation?') + + maps = maps.mean(0)[:, 0] + maps = maps[:len(self.pipe.tokenizer.tokenize(prompt)) + 2] # 1 for SOS and 1 for padding + + if normalize: + maps = maps / (maps[1:-1].sum(0, keepdim=True) + 1e-6) # drop out [SOS] and [PAD] for proper probabilities + + return GlobalHeatMap(self.pipe.tokenizer, prompt, maps) + + +class ImageProcessorHooker(ObjectHooker[VaeImageProcessor]): + def __init__(self, processor: VaeImageProcessor, parent_trace: 'trace'): + super().__init__(processor) + self.parent_trace = parent_trace + + def _hooked_postprocess(hk_self, _: VaeImageProcessor, *args, **kwargs): + images = hk_self.monkey_super('postprocess', *args, **kwargs) + hk_self.parent_trace.last_image = images[0] + + return images + + def _hook_impl(self): + self.monkey_patch('postprocess', self._hooked_postprocess) + + +class PipelineHooker(ObjectHooker[StableDiffusionPipeline]): + def __init__(self, pipeline: StableDiffusionPipeline, parent_trace: 'trace'): + super().__init__(pipeline) + self.heat_maps = parent_trace.all_heat_maps + self.parent_trace = parent_trace + + def _hooked_run_safety_checker(hk_self, self: StableDiffusionPipeline, image, *args, **kwargs): + image, has_nsfw = hk_self.monkey_super('run_safety_checker', image, *args, **kwargs) + + if self.image_processor: + if torch.is_tensor(image): + images = self.image_processor.postprocess(image, output_type='pil') + else: + images = self.image_processor.numpy_to_pil(image) + else: + images = self.numpy_to_pil(image) + + hk_self.parent_trace.last_image = images[len(images)-1] + + return image, has_nsfw + + def _hooked_check_inputs(hk_self, _: StableDiffusionPipeline, prompt: Union[str, List[str]], *args, **kwargs): + if not isinstance(prompt, str) and len(prompt) > 1: + raise ValueError('Only single prompt generation is supported for heat map computation.') + elif not isinstance(prompt, str): + last_prompt = prompt[0] + else: + last_prompt = prompt + + hk_self.heat_maps.clear() + hk_self.parent_trace.last_prompt = last_prompt + + return hk_self.monkey_super('check_inputs', prompt, *args, **kwargs) + + def _hook_impl(self): + self.monkey_patch('run_safety_checker', self._hooked_run_safety_checker, strict=False) # not present in SDXL + self.monkey_patch('check_inputs', self._hooked_check_inputs) + + +class UNetCrossAttentionHooker(ObjectHooker[Attention]): + def __init__( + self, + module: Attention, + parent_trace: 'trace', + context_size: int = 77, + layer_idx: int = 0, + latent_hw: int = 9216, + load_heads: bool = False, + save_heads: bool = False, + data_dir: Union[str, Path] = None, + ): + super().__init__(module) + self.heat_maps = parent_trace.all_heat_maps + self.context_size = context_size + self.layer_idx = layer_idx + self.latent_hw = latent_hw + + self.load_heads = load_heads + self.save_heads = save_heads + self.trace = parent_trace + + if data_dir is not None: + data_dir = Path(data_dir) + else: + data_dir = cache_dir() / 'heads' + + self.data_dir = data_dir + self.data_dir.mkdir(parents=True, exist_ok=True) + + @torch.no_grad() + def _unravel_attn(self, x): + # type: (torch.Tensor) -> torch.Tensor + # x shape: (heads, height * width, tokens) + """ + Unravels the attention, returning it as a collection of heat maps. + + Args: + x (`torch.Tensor`): cross attention slice/map between the words and the tokens. + value (`torch.Tensor`): the value tensor. + + Returns: + `List[Tuple[int, torch.Tensor]]`: the list of heat maps across heads. + """ + h = w = int(math.sqrt(x.size(1))) + maps = [] + x = x.permute(2, 0, 1) + + with auto_autocast(dtype=torch.float32): + for map_ in x: + map_ = map_.view(map_.size(0), h, w) + # For Instruct Pix2Pix, divide the map into three parts: text condition, image condition and unconditional, + # and only keep the text condition part, which is first of the three parts(as per diffusers implementation). + if map_.size(0) == 24: + map_ = map_[:((map_.size(0) // 3)+1)] # Filter out unconditional and image condition + else: + map_ = map_[map_.size(0) // 2:] # # Filter out unconditional + maps.append(map_) + + maps = torch.stack(maps, 0) # shape: (tokens, heads, height, width) + return maps.permute(1, 0, 2, 3).contiguous() # shape: (heads, tokens, height, width) + + def _save_attn(self, attn_slice: torch.Tensor): + torch.save(attn_slice, self.data_dir / f'{self.trace._gen_idx}.pt') + + def _load_attn(self) -> torch.Tensor: + return torch.load(self.data_dir / f'{self.trace._gen_idx}.pt') + + def __call__( + self, + attn: Attention, + hidden_states, + encoder_hidden_states=None, + attention_mask=None, + ): + """Capture attentions and aggregate them.""" + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross is not None: + encoder_hidden_states = attn.norm_cross(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + + # DAAM save heads + if self.save_heads: + self._save_attn(attention_probs) + elif self.load_heads: + attention_probs = self._load_attn() + + # compute shape factor + factor = int(math.sqrt(self.latent_hw // attention_probs.shape[1])) + self.trace._gen_idx += 1 + + # skip if too large + if attention_probs.shape[-1] == self.context_size and factor != 8: + # shape: (batch_size, 64 // factor, 64 // factor, 77) + maps = self._unravel_attn(attention_probs) + + for head_idx, heatmap in enumerate(maps): + self.heat_maps.update(factor, self.layer_idx, head_idx, heatmap) + + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + def _hook_impl(self): + self.original_processor = self.module.processor + self.module.set_processor(self) + + def _unhook_impl(self): + self.module.set_processor(self.original_processor) + + @property + def num_heat_maps(self): + return len(next(iter(self.heat_maps.values()))) + + +trace: Type[DiffusionHeatMapHooker] = DiffusionHeatMapHooker diff --git a/scripts/daam/utils.py b/scripts/daam/utils.py new file mode 100644 index 000000000..15e2c1b5e --- /dev/null +++ b/scripts/daam/utils.py @@ -0,0 +1,107 @@ +from functools import lru_cache +from pathlib import Path +import os +import sys +import random +from typing import TypeVar + +import PIL.Image +import matplotlib.pyplot as plt +import numpy as np +import spacy +import torch + + +__all__ = ['set_seed', 'compute_token_merge_indices', 'plot_mask_heat_map', 'cached_nlp', 'cache_dir', 'auto_device', 'auto_autocast'] + + +T = TypeVar('T') + + +def auto_device(obj: T = torch.device('cpu')) -> T: + if isinstance(obj, torch.device): + return torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + if torch.cuda.is_available(): + return obj.to('cuda') + + return obj + + +def auto_autocast(*args, **kwargs): + if not torch.cuda.is_available(): + kwargs['enabled'] = False + + return torch.cuda.amp.autocast(*args, **kwargs) + + +def plot_mask_heat_map(im: PIL.Image.Image, heat_map: torch.Tensor, threshold: float = 0.4): + im = torch.from_numpy(np.array(im)).float() / 255 + mask = (heat_map.squeeze() > threshold).float() + im = im * mask.unsqueeze(-1) + plt.imshow(im) + + +def set_seed(seed: int) -> torch.Generator: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + gen = torch.Generator(device=auto_device()) + gen.manual_seed(seed) + + return gen + + +def cache_dir() -> Path: + # *nix + if os.name == 'posix' and sys.platform != 'darwin': + xdg = os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')) + return Path(xdg, 'daam') + elif sys.platform == 'darwin': + # Mac OS + return Path(os.path.expanduser('~'), 'Library/Caches/daam') + else: + # Windows + local = os.environ.get('LOCALAPPDATA', None) \ + or os.path.expanduser('~\\AppData\\Local') + return Path(local, 'daam') + + +def compute_token_merge_indices(tokenizer, prompt: str, word: str, word_idx: int = None, offset_idx: int = 0): + merge_idxs = [] + tokens = tokenizer.tokenize(prompt.lower()) + tokens = [x.replace('', '') for x in tokens] # New tokenizer uses wordpiece markers. + + if word_idx is None: + word = word.lower() + search_tokens = [x.replace('', '') for x in tokenizer.tokenize(word)] # New tokenizer uses wordpiece markers. + start_indices = [x + offset_idx for x in range(len(tokens)) if tokens[x:x + len(search_tokens)] == search_tokens] + + for indice in start_indices: + merge_idxs += [i + indice for i in range(0, len(search_tokens))] + + if not merge_idxs: + raise ValueError(f'Search word {word} not found in prompt!') + else: + merge_idxs.append(word_idx) + + return [x + 1 for x in merge_idxs], word_idx # Offset by 1. + + +nlp = None + + +@lru_cache(maxsize=100000) +def cached_nlp(prompt: str, type='en_core_web_md'): + global nlp + + if nlp is None: + try: + nlp = spacy.load(type) + except OSError: + os.system(f'python -m spacy download {type}') + nlp = spacy.load(type) + + return nlp(prompt) diff --git a/scripts/daam_ext.py b/scripts/daam_ext.py new file mode 100644 index 000000000..5ae16a855 --- /dev/null +++ b/scripts/daam_ext.py @@ -0,0 +1,57 @@ +# https://github.com/genforce/ctrl-x + +import gradio as gr +from installer import install +from modules import shared, scripts_manager, processing + + +COLORMAP = ['autumn', 'bone', 'jet', 'winter', 'rainbow', 'ocean', 'summer', 'spring', 'cool', 'hsv', 'pink', 'hot', 'parula', 'magma', 'inferno', 'plasma', 'viridis', 'cividis', 'twilight', 'shifted', 'turbo', 'deepgreen'] + + +class Script(scripts_manager.Script): + def title(self): + return 'DAAM: Diffusion Attentive Attribution Maps' + + def show(self, is_img2img): + return not is_img2img + + def ui(self, _is_img2img): + with gr.Row(): + gr.HTML('  DAAM: Diffusion Attentive Attribution Maps
') + with gr.Row(): + append_images = gr.Checkbox(label='Append heatmaps to results', value=True, elem_id='daam_append_images') + colormap = gr.Dropdown(label='Colormap', choices=COLORMAP, value='jet', type='value', elem_id='daam_colormap') + return append_images, colormap + + def run(self, p: processing.StableDiffusionProcessing, append_images, colormap): # pylint: disable=arguments-differ + c = shared.sd_model.__class__.__name__ if shared.sd_loaded else '' + if shared.sd_model_type != 'sdxl': + shared.log.warning(f'DAAM: pipeline={c} required=StableDiffusionXLPipeline') + return None + + install('thinc==8.3.4') + install('spacy==3.8.4') + + from scripts import daam # pylint: disable=no-name-in-module + orig_prompt_attention = shared.opts.prompt_attention + shared.opts.data['prompt_attention'] = 'fixed' + + # process + with daam.trace(shared.sd_model) as tc: + processed: processing.Processed = processing.process_images(p) + global_heat_map = tc.compute_global_heat_map() + shared.log.info(f'DAAM: prompt="{global_heat_map.prompt}" heatmaps={global_heat_map.heat_maps.shape}') + + # word_heat_map = global_heat_map.compute_word_heat_map('woman') + parsed_heat_maps = global_heat_map.parsed_heat_maps() + if append_images: + image = processed.images[0] + for parsed_heat_map in parsed_heat_maps: + if len(parsed_heat_map.token.text) > 1: + shared.log.debug(f'DAAM: token="{parsed_heat_map.token.text}"') + overlay = parsed_heat_map.word_heat_map.plot_overlay(image=image, color_normalize=True, cmap=colormap) + processed.images.append(overlay) + + # restore and return + shared.opts.data['prompt_attention'] = orig_prompt_attention + return processed diff --git a/scripts/infiniteyou_ext.py b/scripts/infiniteyou_ext.py index e38185aa4..677d56cca 100644 --- a/scripts/infiniteyou_ext.py +++ b/scripts/infiniteyou_ext.py @@ -13,11 +13,9 @@ orig_pipeline, orig_prompt_attention = None, None def verify_insightface(): - from installer import installed, install, install_pydantic + from installer import installed, install_insightface if not installed('insightface', reload=False, quiet=True): - install('git+https://github.com/deepinsight/insightface@554a05561cb71cfebb4e012dfea48807f845a0c2#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches - install('albumentations==1.4.3', ignore=False, reinstall=True) - install_pydantic() + install_insightface() def load_infiniteyou(model: str): diff --git a/scripts/loopback.py b/scripts/loopback.py index a22b153a2..e24683ce6 100644 --- a/scripts/loopback.py +++ b/scripts/loopback.py @@ -1,4 +1,5 @@ import math +import random import gradio as gr from modules import images, processing, scripts_manager @@ -21,10 +22,12 @@ class Script(scripts_manager.Script): final_denoising_strength = gr.Slider(minimum=0, maximum=1, step=0.01, label='Final strength', value=0.5, elem_id=self.elem_id("final_denoising_strength")) with gr.Row(): denoising_curve = gr.Dropdown(label="Strength curve", choices=["Aggressive", "Linear", "Lazy"], value="Linear") + with gr.Row(): + randomize_seed = gr.Checkbox(label="Randomize seed after each loop iteration", value=False) - return [loops, final_denoising_strength, denoising_curve] + return [loops, final_denoising_strength, denoising_curve, randomize_seed] - def run(self, p, loops, final_denoising_strength, denoising_curve): # pylint: disable=arguments-differ + def run(self, p, loops, final_denoising_strength, denoising_curve, randomize_seed): # pylint: disable=arguments-differ processing.fix_seed(p) initial_batch_count = p.n_iter p.extra_generation_params['Loopback'] = final_denoising_strength @@ -76,10 +79,14 @@ class Script(scripts_manager.Script): if initial_seed is None: initial_seed = processed.seed initial_info = processed.info + if randomize_seed: + p.seed = random.randrange(4294967294) + p.all_seeds = [p.seed] p.seed = processed.seed + 1 # why? p.denoising_strength = calculate_denoising_strength(i + 1) last_image = processed.images[0] p.init_images = [last_image] + log.info(f'Loopback: iteration={i} seed={p.seed} curve={denoising_curve} strength={p.denoising_strength}:{final_denoising_strength}') if initial_batch_count == 1: history.append(last_image) all_images.append(last_image) diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index aea0e5730..221d815dc 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -159,7 +159,7 @@ class Script(scripts_manager.Script): trust_remote_code=True, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, - _attn_implementation="eager", + # _attn_implementation="eager", **gguf_args, **quant_args, ) diff --git a/scripts/prompts_from_file.py b/scripts/prompts_from_file.py index 460b39ecf..ae2e8038f 100644 --- a/scripts/prompts_from_file.py +++ b/scripts/prompts_from_file.py @@ -147,4 +147,6 @@ class Script(scripts_manager.Script): all_negative += proc.all_negative_prompts images += proc.images infotexts += proc.infotexts + if state.interrupted: + break return get_processed(p, images, p.seed, "", all_prompts=all_prompts, all_seeds=all_seeds, all_negative_prompts=all_negative, infotexts=infotexts) diff --git a/scripts/pulid/__init__.py b/scripts/pulid/__init__.py index 958ee9c1e..864b758d1 100644 --- a/scripts/pulid/__init__.py +++ b/scripts/pulid/__init__.py @@ -8,6 +8,7 @@ from modules.errors import log sys.path.append(os.path.dirname(__file__)) try: from pulid_sdxl import StableDiffusionXLPuLIDPipeline, StableDiffusionXLPuLIDPipelineImage, StableDiffusionXLPuLIDPipelineInpaint + from pulid_flux import apply_flux, unapply_flux from pulid_utils import resize_numpy_image_long as resize import attention_processor as attention import pulid_sampling as sampling diff --git a/scripts/pulid/pulid_flux.py b/scripts/pulid/pulid_flux.py new file mode 100644 index 000000000..a419ca9c0 --- /dev/null +++ b/scripts/pulid/pulid_flux.py @@ -0,0 +1,35 @@ +from types import MethodType +import accelerate +from diffusers import FluxPipeline +from modules import shared, sd_models + + +def apply_flux(pipe: FluxPipeline): + if not hasattr(pipe, 'transformer') or not 'Nunchaku' in pipe.transformer.__class__.__name__: + shared.log.error('PuLID: flux support requires nunchaku') + return pipe + + from nunchaku.pipeline.pipeline_flux_pulid import PuLIDFluxPipeline + if not isinstance(pipe, PuLIDFluxPipeline): + from nunchaku.models.pulid.pulid_forward import pulid_forward + sd_models.clear_caches(full=True) + accelerate.hooks.remove_hook_from_module(pipe.transformer, recurse=True) + pipe = sd_models.switch_pipe(PuLIDFluxPipeline, pipe) + pipe.transformer.orig_forward = pipe.transformer.forward + pipe.transformer.forward = MethodType(pulid_forward, pipe.transformer) + pipe = sd_models.apply_balanced_offload(pipe) + pipe.pulid_model = sd_models.apply_balanced_offload(pipe.pulid_model) + shared.log.info(f'PuLID: flux applied cls={pipe.__class__.__name__} pipe={pipe.pulid_model.__class__.__name__}') + return pipe + + +def unapply_flux(pipe: FluxPipeline): + from nunchaku.pipeline.pipeline_flux_pulid import PuLIDFluxPipeline + if isinstance(pipe, PuLIDFluxPipeline) and hasattr(pipe.transformer, 'orig_forward'): + sd_models.clear_caches(full=True) + accelerate.hooks.remove_hook_from_module(pipe.transformer, recurse=True) + pipe.transformer.forward = MethodType(pipe.transformer.orig_forward, pipe.transformer) + del pipe.transformer.orig_forward + pipe = sd_models.switch_pipe(FluxPipeline, pipe) + pipe = sd_models.apply_balanced_offload(pipe) + return pipe diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index a92480824..ede691902 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -28,11 +28,9 @@ class Script(scripts_manager.Script): return True def dependencies(self): - from installer import install, installed, install_pydantic - if not installed('insightface==0.7.3', reload=False, quiet=True): - install('git+https://github.com/deepinsight/insightface@554a05561cb71cfebb4e012dfea48807f845a0c2#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches - install('albumentations==1.4.3', ignore=False, reinstall=True) - install_pydantic() + from installer import installed, install_insightface + if not installed('insightface', reload=False, quiet=True): + install_insightface() def register(self): # register xyz grid elements global registered # pylint: disable=global-statement @@ -202,14 +200,12 @@ class Script(scripts_manager.Script): errors.display(e, 'PuLID') return None elif shared.sd_model_type == 'f1': - # TODO nunchaku: pulid-f1 - shared.log.error('PuLID: f1 not supported') - return None + shared.sd_model = self.pulid.apply_flux(shared.sd_model) if shared.sd_model_type == 'sdxl': processed = self.run_sdxl(p, images, strength, zero, sampler, ortho, restore, offload, version) elif shared.sd_model_type == 'f1': - processed = None + processed = self.run_flux(p, images, strength) else: shared.log.error(f'PuLID: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={supported_model_list}') processed = None @@ -229,6 +225,12 @@ class Script(scripts_manager.Script): shared.sd_model = shared.sd_model.pipe devices.torch_gc(force=True, reason='pulid') shared.log.debug(f'PuLID complete: class={shared.sd_model.__class__.__name__} preprocess={self.preprocess:.2f} pipe={"restore" if restore else "cache"}') + if shared.sd_model_type == "f1": + restore = getattr(p, 'pulid_restore', restore) + if restore: + shared.sd_model = self.pulid.unapply_flux(shared.sd_model) + devices.torch_gc(force=True, reason='pulid') + shared.log.debug(f'PuLID complete: class={shared.sd_model.__class__.__name__} pipe={"restore" if restore else "cache"}') return processed def run_sdxl(self, p: processing.StableDiffusionProcessing, images: list, strength: float, zero: int, sampler: str, ortho: str, restore: bool, offload: bool, version: str): @@ -287,3 +289,13 @@ class Script(scripts_manager.Script): # interim = [Image.fromarray(img) for img in shared.sd_model.debug_img_list] # shared.log.debug(f'PuLID: time={t1-t0:.2f}') return processed + + def run_flux(self, p: processing.StableDiffusionProcessing, images: list, strength: float): + image = Image.fromarray(images[0]) # takes single pil image + p.task_args['id_image'] = image + p.task_args['id_weight'] = strength + shared.log.info(f'PuLID: class={shared.sd_model.__class__.__name__} strength={strength} image={image}') + p.extra_generation_params["PuLID"] = f'Strength={strength}' + + processed: processing.Processed = processing.process_images(p) # runs processing using main loop + return processed diff --git a/scripts/xyz/xyz_grid_classes.py b/scripts/xyz/xyz_grid_classes.py index 306b591f3..0836adbbf 100644 --- a/scripts/xyz/xyz_grid_classes.py +++ b/scripts/xyz/xyz_grid_classes.py @@ -20,6 +20,7 @@ from scripts.xyz.xyz_grid_shared import ( # pylint: disable=no-name-in-module, u apply_lora, apply_lora_strength, apply_te, + apply_guidance, apply_styles, apply_upscaler, apply_context, @@ -223,6 +224,7 @@ axis_options = [ AxisOption("[Sampler] Shift", float, apply_setting("schedulers_shift")), AxisOption("[Sampler] eta delta", float, apply_setting("eta_noise_seed_delta")), AxisOption("[Sampler] eta multiplier", float, apply_setting("scheduler_eta")), + AxisOption("[Guidance] Guidance name", str, apply_guidance, choices=lambda: ['Default', 'CFG', 'Auto', 'Zero', 'PAG', 'APG', 'SLG', 'SEG', 'TCFG', 'FDG']), AxisOption("[Refine] Upscaler", str, apply_field("hr_upscaler"), cost=0.3, choices=lambda: [x.name for x in shared.sd_upscalers]), AxisOption("[Refine] Sampler", str, apply_hr_sampler_name, fmt=format_value_add_label, confirm=confirm_samplers, choices=lambda: [x.name for x in sd_samplers.samplers]), AxisOption("[Refine] Denoising strength", float, apply_field("denoising_strength")), diff --git a/scripts/xyz/xyz_grid_shared.py b/scripts/xyz/xyz_grid_shared.py index 0c95d14de..76d4df769 100644 --- a/scripts/xyz/xyz_grid_shared.py +++ b/scripts/xyz/xyz_grid_shared.py @@ -254,6 +254,13 @@ def apply_te(p, x, xs): shared.log.debug(f'XYZ grid apply text-encoder: "{x}"') +def apply_guidance(p, x, xs): + from modules.modular_guiders import guiders + guiders = list(guiders.keys()) + p.guidance_name = [g for g in guiders if g.lower().startswith(x.lower())][0] + shared.log.debug(f'XYZ grid apply guidance: "{p.guidance_name}"') + + def apply_styles(p: processing.StableDiffusionProcessingTxt2Img, x: str, _): p.styles.extend(x.split(',')) shared.log.debug(f'XYZ grid apply style: "{x}"') diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index 2ec2452da..3874deb5e 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -54,9 +54,9 @@ class Script(scripts_manager.Script): with gr.Row(): with gr.Column(): - csv_mode = gr.Checkbox(label='Text inputs', value=False, elem_id=self.elem_id("csv_mode"), container=False) - draw_legend = gr.Checkbox(label='Legend', value=True, elem_id=self.elem_id("draw_legend"), container=False) - no_fixed_seeds = gr.Checkbox(label='Random seeds', value=False, elem_id=self.elem_id("no_fixed_seeds"), container=False) + draw_legend = gr.Checkbox(label='Draw legend', value=True, elem_id=self.elem_id("draw_legend"), container=False) + csv_mode = gr.Checkbox(label='Use text inputs', value=False, elem_id=self.elem_id("csv_mode"), container=False) + no_fixed_seeds = gr.Checkbox(label='Use random seeds', value=False, elem_id=self.elem_id("no_fixed_seeds"), container=False) include_time = gr.Checkbox(label='Add time info', value=False, elem_id=self.elem_id("include_time"), container=False) include_text = gr.Checkbox(label='Add text info', value=False, elem_id=self.elem_id("include_text"), container=False) with gr.Column(): @@ -168,8 +168,6 @@ class Script(scripts_manager.Script): create_video, video_type, video_duration, video_loop, video_pad, video_interpolate, ): # pylint: disable=W0221 jobid = shared.state.begin('XYZ Grid') - if not no_fixed_seeds: - processing.fix_seed(p) if not shared.opts.return_grid: p.batch_size = 1 @@ -251,6 +249,8 @@ class Script(scripts_manager.Script): xs = fix_axis_seeds(x_opt, xs) ys = fix_axis_seeds(y_opt, ys) zs = fix_axis_seeds(z_opt, zs) + else: + processing.fix_seed(p) total_jobs = len(xs) * len(ys) * len(zs) if x_opt.label == 'Steps': diff --git a/scripts/xyz_grid_on.py b/scripts/xyz_grid_on.py index f5a910f93..595536067 100644 --- a/scripts/xyz_grid_on.py +++ b/scripts/xyz_grid_on.py @@ -324,11 +324,17 @@ class Script(scripts_manager.Script): def cell(x, y, z, ix, iy, iz): if shared.state.interrupted: + shared.log.warning('XYZ grid: Interrupted') return processing.Processed(p, [], p.seed, ""), 0 p.xyz = True pc = copy(p) pc.override_settings_restore_afterwards = False pc.styles = pc.styles[:] + if no_fixed_seeds: + pc.seed = -1 + processing.fix_seed(pc) + pc.all_seeds = None + pc.all_subseeds = None x_opt.apply(pc, x, xs) y_opt.apply(pc, y, ys) z_opt.apply(pc, z, zs) diff --git a/webui.py b/webui.py index ec33c908e..8dc29d5bc 100644 --- a/webui.py +++ b/webui.py @@ -152,9 +152,9 @@ def initialize(): def load_model(): modeldata.model_data.locked = False - if not shared.opts.sd_checkpoint_autoload and shared.cmd_opts.ckpt is None: - log.info('Model: autoload=False') - else: + autoload = shared.opts.sd_checkpoint_autoload or shared.cmd_opts.ckpt is not None + log.info(f'Model: autoload={autoload} selected="{shared.opts.sd_model_checkpoint}"') + if autoload: jobid = shared.state.begin('Load model') thread_model = Thread(target=lambda: shared.sd_model) thread_model.start() diff --git a/wiki b/wiki index 76be4e54b..f274489e4 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 76be4e54b2c40bac1502cf97caa4a04bec0a033f +Subproject commit f274489e42b97b57ea91b9e94087b8b39820a4ee